New AWS experiment and updated K8 experiment (#297)

* New experiment for AWS and update on K8

Signed-off-by: Sumit Nagal <sumit_nagal@intuit.com>
This commit is contained in:
sumit Nagal
2020-08-18 23:29:39 -07:00
committed by GitHub
parent 7526cdb529
commit 3e50e15497
28 changed files with 812 additions and 0 deletions

View File

@@ -0,0 +1,52 @@
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
name: aws-ec2-terminate
namespace: default
spec:
appinfo:
appns: 'default'
applabel: 'app=nginx'
appkind: 'deployment'
annotationCheck: 'false'
engineState: 'active'
monitoring: false
jobCleanUpPolicy: 'retain'
chaosServiceAccount: chaos-admin
components:
runner:
runnerannotation:
iam.amazonaws.com/role: "k8s-chaosec2access"
experiments:
- name: aws-ec2-terminate
spec:
components:
experimentannotation:
iam.amazonaws.com/role: "k8s-chaosec2access"
env:
- name: NAME_SPACE
value: default
- name: LABEL_NAME
value: app=nginx
- name: APP_ENDPOINT
value: localhost
- name: FILE
value: 'ec2-delete.json'
- name: AWS_ROLE
value: 'chaosec2access'
- name: AWS_ACCOUNT
value: '0000000000'
- name: AWS_REGION
value: 'us-west-2'
- name: AWS_AZ
value: 'us-west-2c'
- name: AWS_RESOURCE
value: 'ec2-iks'
- name: AWS_SSL
value: 'false'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -0,0 +1,105 @@
apiVersion: litmuschaos.io/v1alpha1
description:
message: |
Deletes an aws instance belonging to a deployment/statefulset/daemonset
kind: ChaosExperiment
metadata:
name: aws-ec2-terminate
version: 0.0.1
spec:
definition:
scope: Namespaced
permissions:
- apiGroups:
- ""
- "apps"
- "batch"
- "litmuschaos.io"
resources:
- "deployments"
- "jobs"
- "pods"
- "configmaps"
- "chaosengines"
- "chaosexperiments"
- "chaosresults"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "update"
- "delete"
- apiGroups:
- ""
resources:
- "nodes"
verbs :
- "get"
- "list"
image: "litmuschaos/chaostoolkit:latest"
args:
- -c
- python /app/chaos/chaostest/aws/aws_wrapper.py ; exit 0
command:
- /bin/bash
env:
- name: CHAOSTOOLKIT_IN_POD
value: 'true'
- name: FILE
value: 'ec2-delete.json'
- name: NAME_SPACE
value: 'default'
- name: LABEL_NAME
value: 'app=nginx'
- name: APP_ENDPOINT
value: 'localhost'
# Period to wait before injection of chaos in sec
- name: PERCENTAGE
value: '50'
# Variable to set for custom report upload
- name: REPORT
value: 'false'
# Variable to set for report upload endpoint
- name: REPORT_ENDPOINT
value: 'none'
# Variable to set for AWS account
- name: AWS_ACCOUNT
value: '000000000000'
# Variable to set for AWS role, Make sure you have created this role and have give access
- name: AWS_ROLE
value: 'chaosec2access'
# Variable to set for AWS region
- name: AWS_REGION
value: 'us-west-2'
# Variable to set for AWS AZ
- name: AWS_AZ
value: 'us-west-2c'
# Variable to set for AWS RESOURCE
- name: AWS_RESOURCE
value: 'ec2-iks'
# Variable to set for AWS SSL
- name: AWS_SSL
value: 'false'
# Variable which indicates where the test results CRs will be persisted
- name: TEST_NAMESPACE
value: 'default'
labels:
name: aws-ec2-terminate

View File

@@ -0,0 +1,35 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: chaos-admin
labels:
name: chaos-admin
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: chaos-admin
labels:
name: chaos-admin
rules:
- apiGroups: ["","apps","batch","extensions","litmuschaos.io","openebs.io","storage.k8s.io"]
resources: ["chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/exec","pods/log","secrets","storageclasses","chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","daemonsets","deployments","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/eviction","pods/exec","pods/log","replicasets","secrets","services","statefulsets","storageclasses"]
verbs: ["create","delete","get","list","patch","update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get","list","patch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: chaos-admin
labels:
name: chaos-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: chaos-admin
subjects:
- kind: ServiceAccount
name: chaos-admin
namespace: default

View File

@@ -0,0 +1,16 @@
# In Namespace Changes
# This experiment assume that you are using AWS with kubernetes
# This experiment assume your namespace has right role for aws to make aws api calls
# This experiment also assume you are using Instance group for your name space or aware that if you are using share
# node group, it will impact other pods running on this ec2 instance
* Apply experiments for k8 - `kubectl apply -f experiments.yaml`
* Validate the experiments for k8 - `kubectl get chaosexperiment`
* Setup RBAC - for pod delete RBAC - `kubectl apply -f rbac.yaml`
* Create pod Experiment - for health experiment -`kubectl create -f engine.yaml`
* Validate experiment - `kubectl get pods -o wide`
* Validate logs - `kubectl logs -f <delete pod>`
* Clean up chaosexperiment -`kubectl delete -f engine.yaml`
* Clean up rbac -`kubectl delete -f rbac.yaml`

View File

@@ -0,0 +1,35 @@
apiVersion: litmuchaos.io/v1alpha1
kind: ChartServiceVersion
metadata:
name: aws-ec2-terminate
version: 0.0.1
annotations:
categories: Kubernetes
vendor: CNCF
createdAt: 2020-02-24T10:28:08Z
support: https://slack.kubernetes.io/
spec:
displayName: aws-ec2-terminate
categoryDescription: |
AWS EC2 terminate contains chaos to disrupt state of aws resources running part of kuberntes cluster workload. It uses chaostoolkit to inject random pod delete failures against specified applications
keywords:
- Kubernetes
- AWS
- EC2
- State
platforms:
- Minikube
maturity: alpha
maintainers:
- name: sumit
email: sumit_nagal@intuit.com
minKubeVersion: 1.12.0
provider:
name: Intuit
links:
- name: Source Code
url: https://github.com/litmuschaos/litmus-python/tree/master/chaos-test
icon:
- url:
mediatype: ""
chaosexpcrdlink: https://raw.githubusercontent.com/litmuschaos/chaos-charts/master/charts/generic/k8-aws-ec2-terminate/experiment.yaml

View File

@@ -0,0 +1,52 @@
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
name: aws-ec2-terminate
namespace: default
spec:
appinfo:
appns: 'default'
applabel: 'app=nginx'
appkind: 'deployment'
annotationCheck: 'false'
engineState: 'active'
monitoring: false
jobCleanUpPolicy: 'retain'
chaosServiceAccount: chaos-admin
components:
runner:
runnerannotation:
iam.amazonaws.com/role: "k8s-chaosec2access"
experiments:
- name: aws-ec2-terminate
spec:
components:
experimentannotation:
iam.amazonaws.com/role: "k8s-chaosec2access"
env:
- name: NAME_SPACE
value: default
- name: LABEL_NAME
value: app=nginx
- name: APP_ENDPOINT
value: localhost
- name: FILE
value: 'ec2-delete.json'
- name: AWS_ROLE
value: 'chaosec2access'
- name: AWS_ACCOUNT
value: '0000000000'
- name: AWS_REGION
value: 'us-west-2'
- name: AWS_AZ
value: 'us-west-2c'
- name: AWS_RESOURCE
value: 'ec2-iks'
- name: AWS_SSL
value: 'false'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -0,0 +1,105 @@
apiVersion: litmuschaos.io/v1alpha1
description:
message: |
Deletes an aws instance belonging to a deployment/statefulset/daemonset
kind: ChaosExperiment
metadata:
name: aws-ec2-terminate
version: 0.0.1
spec:
definition:
scope: Namespaced
permissions:
- apiGroups:
- ""
- "apps"
- "batch"
- "litmuschaos.io"
resources:
- "deployments"
- "jobs"
- "pods"
- "configmaps"
- "chaosengines"
- "chaosexperiments"
- "chaosresults"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "update"
- "delete"
- apiGroups:
- ""
resources:
- "nodes"
verbs :
- "get"
- "list"
image: "litmuschaos/chaostoolkit:latest"
args:
- -c
- python /app/chaos/chaostest/aws/aws_wrapper.py ; exit 0
command:
- /bin/bash
env:
- name: CHAOSTOOLKIT_IN_POD
value: 'true'
- name: FILE
value: 'ec2-delete.json'
- name: NAME_SPACE
value: 'default'
- name: LABEL_NAME
value: 'app=nginx'
- name: APP_ENDPOINT
value: 'localhost'
# Period to wait before injection of chaos in sec
- name: PERCENTAGE
value: '50'
# Variable to set for custom report upload
- name: REPORT
value: 'false'
# Variable to set for report upload endpoint
- name: REPORT_ENDPOINT
value: 'none'
# Variable to set for AWS account
- name: AWS_ACCOUNT
value: '000000000000'
# Variable to set for AWS role, Make sure you have created this role and have give access
- name: AWS_ROLE
value: 'chaosec2access'
# Variable to set for AWS region
- name: AWS_REGION
value: 'us-west-2'
# Variable to set for AWS AZ
- name: AWS_AZ
value: 'us-west-2c'
# Variable to set for AWS RESOURCE
- name: AWS_RESOURCE
value: 'ec2-iks'
# Variable to set for AWS SSL
- name: AWS_SSL
value: 'false'
# Variable which indicates where the test results CRs will be persisted
- name: TEST_NAMESPACE
value: 'default'
labels:
name: aws-ec2-terminate

View File

@@ -0,0 +1,35 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: chaos-admin
labels:
name: chaos-admin
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: chaos-admin
labels:
name: chaos-admin
rules:
- apiGroups: ["","apps","batch","extensions","litmuschaos.io","openebs.io","storage.k8s.io"]
resources: ["chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/exec","pods/log","secrets","storageclasses","chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","daemonsets","deployments","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/eviction","pods/exec","pods/log","replicasets","secrets","services","statefulsets","storageclasses"]
verbs: ["create","delete","get","list","patch","update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get","list","patch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: chaos-admin
labels:
name: chaos-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: chaos-admin
subjects:
- kind: ServiceAccount
name: chaos-admin
namespace: default

View File

@@ -34,3 +34,6 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -34,3 +34,6 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -34,3 +34,6 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -34,3 +34,6 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,7 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -33,4 +33,6 @@ spec:
value: 'true' value: 'true'
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -67,6 +67,10 @@ spec:
- name: REPORT_ENDPOINT - name: REPORT_ENDPOINT
value: 'none' value: 'none'
- name: TEST_NAMESPACE
value: 'default'
labels: labels:
name: k8-pod-delete name: k8-pod-delete

View File

@@ -0,0 +1,10 @@
# Remote namespace
# This experiment help you to kill a micro service running on the k8 cluster
* Apply experiments for K8 - `kubectl apply -f experiments.yaml`
* Validate the experiments for k8 - `kubectl get chaosexperiments`
* Setup RBAC as admin mode - `kubectl apply -f rbac.yaml`
* Create pod Experiment - for health experiment for IKS 1.0 -`kubectl create -f engine-kiam.yaml`
* Validate experiment - `kubectl get pods -w`
* Validate logs - `kubectl logs -f <delete pod>`
* Clean up chaosexperiment -`kubectl delete -f engine.yaml`
* Clean up rbac -`kubectl delete -f rbac.yaml`

View File

@@ -0,0 +1,38 @@
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
name: nginx-chaos-service-health
namespace: default
spec:
appinfo:
appns: 'default'
applabel: 'app=nginx'
appkind: 'deployment'
annotationCheck: 'true'
engineState: 'active'
chaosServiceAccount: chaos-admin
monitoring: false
jobCleanUpPolicy: 'retain'
experiments:
- name: k8-service-kill
spec:
components:
env:
# set chaos namespace
- name: NAME_SPACE
value: 'default'
# set chaos label name
- name: LABEL_NAME
value: 'nginx'
# pod endpoint
- name: APP_ENDPOINT
value: 'localhost'
- name: FILE
value: 'service-app-kill-health.json'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -0,0 +1,76 @@
apiVersion: litmuschaos.io/v1alpha1
description:
message: |
Deletes a pod belonging to a deployment/statefulset/daemonset
kind: ChaosExperiment
metadata:
name: k8-service-kill
version: 0.0.1
spec:
definition:
scope: Namespaced
permissions:
- apiGroups:
- ""
- "apps"
- "batch"
- "litmuschaos.io"
resources:
- "deployments"
- "jobs"
- "pods"
- "configmaps"
- "chaosengines"
- "chaosexperiments"
- "chaosresults"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "update"
- "delete"
- apiGroups:
- ""
resources:
- "nodes"
verbs :
- "get"
- "list"
image: "litmuschaos/chaostoolkit:latest"
args:
- -c
- python /app/chaos/chaostest/kubernetes/k8_wrapper.py; exit 0
command:
- /bin/bash
env:
- name: CHAOSTOOLKIT_IN_POD
value: 'true'
- name: FILE
value: 'service-app-kill-health.json'
- name: NAME_SPACE
value: ''
- name: LABEL_NAME
value: ''
- name: APP_ENDPOINT
value: ''
- name: PERCENTAGE
value: '50'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'
labels:
name: k8-service-kill

View File

@@ -0,0 +1,35 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: chaos-admin
labels:
name: chaos-admin
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: chaos-admin
labels:
name: chaos-admin
rules:
- apiGroups: ["","apps","batch","extensions","litmuschaos.io","openebs.io","storage.k8s.io"]
resources: ["chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/exec","pods/log","secrets","storageclasses","chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","daemonsets","deployments","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/eviction","pods/exec","pods/log","replicasets","secrets","services","statefulsets","storageclasses"]
verbs: ["create","delete","get","list","patch","update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get","list","patch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: chaos-admin
labels:
name: chaos-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: chaos-admin
subjects:
- kind: ServiceAccount
name: chaos-admin
namespace: default

View File

@@ -0,0 +1,38 @@
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
name: nginx-chaos-service-health
namespace: default
spec:
appinfo:
appns: 'default'
applabel: 'app=nginx'
appkind: 'deployment'
annotationCheck: 'true'
engineState: 'active'
chaosServiceAccount: chaos-admin
monitoring: false
jobCleanUpPolicy: 'retain'
experiments:
- name: k8-service-kill
spec:
components:
env:
# set chaos namespace
- name: NAME_SPACE
value: 'default'
# set chaos label name
- name: LABEL_NAME
value: 'nginx'
# pod endpoint
- name: APP_ENDPOINT
value: 'localhost'
- name: FILE
value: 'service-app-kill-health.json'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'

View File

@@ -0,0 +1,76 @@
apiVersion: litmuschaos.io/v1alpha1
description:
message: |
Deletes a pod belonging to a deployment/statefulset/daemonset
kind: ChaosExperiment
metadata:
name: k8-service-kill
version: 0.0.1
spec:
definition:
scope: Namespaced
permissions:
- apiGroups:
- ""
- "apps"
- "batch"
- "litmuschaos.io"
resources:
- "deployments"
- "jobs"
- "pods"
- "configmaps"
- "chaosengines"
- "chaosexperiments"
- "chaosresults"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "update"
- "delete"
- apiGroups:
- ""
resources:
- "nodes"
verbs :
- "get"
- "list"
image: "litmuschaos/chaostoolkit:latest"
args:
- -c
- python /app/chaos/chaostest/kubernetes/k8_wrapper.py; exit 0
command:
- /bin/bash
env:
- name: CHAOSTOOLKIT_IN_POD
value: 'true'
- name: FILE
value: 'service-app-kill-health.json'
- name: NAME_SPACE
value: ''
- name: LABEL_NAME
value: ''
- name: APP_ENDPOINT
value: ''
- name: PERCENTAGE
value: '50'
- name: REPORT
value: 'true'
- name: REPORT_ENDPOINT
value: 'none'
- name: TEST_NAMESPACE
value: 'default'
labels:
name: k8-service-kill

View File

@@ -0,0 +1,33 @@
apiVersion: litmuchaos.io/v1alpha1
kind: ChartServiceVersion
metadata:
name: k8-pod-delete
version: 0.0.4
annotations:
categories: Kubernetes
vendor: CNCF
createdAt: 2020-02-24T10:28:08Z
support: https://slack.kubernetes.io/
spec:
displayName: k8-pod-delete
categoryDescription: |
K8 Pod delete contains chaos to disrupt state of kubernetes resources. It uses chaostoolkit to inject random pod delete failures against specified applications
keywords:
- Kubernetes
- State
platforms:
- Minikube
maturity: alpha
maintainers:
- name: sumit
email: sumit_nagal@intuit.com
minKubeVersion: 1.12.0
provider:
name: Intuit
links:
- name: Source Code
url: https://github.com/litmuschaos/litmus-python/tree/master/chaos-test
icon:
- url:
mediatype: ""
chaosexpcrdlink: https://raw.githubusercontent.com/litmuschaos/chaos-charts/master/charts/generic/k8-pod-delete/experiment.yaml

View File

@@ -0,0 +1,35 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: chaos-admin
labels:
name: chaos-admin
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: chaos-admin
labels:
name: chaos-admin
rules:
- apiGroups: ["","apps","batch","extensions","litmuschaos.io","openebs.io","storage.k8s.io"]
resources: ["chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/exec","pods/log","secrets","storageclasses","chaosengines","chaosexperiments","chaosresults","configmaps","cstorpools","cstorvolumereplicas","daemonsets","deployments","events","jobs","persistentvolumeclaims","persistentvolumes","pods","pods/eviction","pods/exec","pods/log","replicasets","secrets","services","statefulsets","storageclasses"]
verbs: ["create","delete","get","list","patch","update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get","list","patch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: chaos-admin
labels:
name: chaos-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: chaos-admin
subjects:
- kind: ServiceAccount
name: chaos-admin
namespace: default