changing deployment strategy to jobs per pod

blockchain-watcher/config/custom-environment-variables.json

@@ -3,6 +3,9 @@
   "port": "PORT",
   "logLevel": "LOG_LEVEL",
   "dryRun": "DRY_RUN_ENABLED",
+  "jobs": {
+    "dir": "JOBS_DIR"
+  },
   "sns": {
     "topicArn": "SNS_TOPIC_ARN",
     "region": "SNS_REGION"

deploy/blockchain-watcher/env/staging-mainnet.env

@@ -8,9 +8,9 @@ IMAGE_NAME=
 PORT=3005
 LOG_LEVEL=debug
 
-RESOURCES_LIMITS_MEMORY=256Mi
+RESOURCES_LIMITS_MEMORY=128Mi
 RESOURCES_LIMITS_CPU=200m
-RESOURCES_REQUESTS_MEMORY=128Mi
+RESOURCES_REQUESTS_MEMORY=96Mi
 RESOURCES_REQUESTS_CPU=100m
 
 SNS_TOPIC_ARN=

deploy/blockchain-watcher/env/staging-testnet.env

@@ -6,11 +6,11 @@ DRY_RUN_ENABLED=false
 REPLICAS=1
 IMAGE_NAME=
 PORT=3005
-LOG_LEVEL=info
+LOG_LEVEL=debug
 
-RESOURCES_LIMITS_MEMORY=256Mi
+RESOURCES_LIMITS_MEMORY=128Mi
 RESOURCES_LIMITS_CPU=200m
-RESOURCES_REQUESTS_MEMORY=128Mi
+RESOURCES_REQUESTS_MEMORY=96Mi
 RESOURCES_REQUESTS_CPU=100m
 
 SNS_TOPIC_ARN=

deploy/blockchain-watcher/metadata-pv.yaml

@@ -1,27 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: blockchain-watcher-metadata-pvc
-  namespace: {{ .NAMESPACE }}
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Mi
-  storageClassName: gp2
----
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: blockchain-watcher-metadata-pv
-  namespace: {{ .NAMESPACE }}
-spec:
-  accessModes:
-    - ReadWriteOnce
-    - ReadWriteMany
-  capacity:
-    storage: 100Mi
-  storageClassName: gp2
-  hostPath:
-    path: /home/node/app/metadata-repo

deploy/blockchain-watcher/sa.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: event-watcher
+  namespace: {{ .NAMESPACE }}
+  annotations:
+    eks.amazonaws.com/role-arn: {{ .AWS_IAM_ROLE }}

deploy/blockchain-watcher/stateful-set.yaml

@@ -1,71 +0,0 @@
----          
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .NAME }}
-  namespace: {{ .NAMESPACE }}
-  labels:
-    app: {{ .NAME }}
-spec:
-  selector:
-    app: {{ .NAME }}
-  ports:
-    - port: {{ .PORT }}
-      targetPort: {{ .PORT }}
-      name: {{ .NAME }}
-      protocol: TCP
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: {{ .NAME }}
-  namespace: {{ .NAMESPACE }}
-spec:
-  replicas: {{ .REPLICAS }}
-  serviceName: {{ .NAME }}-service
-  selector:
-    matchLabels:
-      app: {{ .NAME }}
-  template:
-    metadata:
-      labels:
-        app: {{ .NAME }}
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "{{ .PORT }}"
-    spec:
-      restartPolicy: Always
-      terminationGracePeriodSeconds: 30
-      serviceAccountName: event-watcher
-      containers:
-        - name: {{ .NAME }}
-          image: {{ .IMAGE_NAME }}
-          env:
-            - name: NODE_ENV
-              value: {{ .NODE_ENV }}
-            - name: PORT
-              value: "{{ .PORT }}"
-            - name: LOG_LEVEL
-              value: {{ .LOG_LEVEL }}
-            - name: BLOCKCHAIN_ENV
-              value: {{ .BLOCKCHAIN_ENV }}
-            - name: DRY_RUN_ENABLED
-              value: "{{ .DRY_RUN_ENABLED }}"
-            - name: SNS_TOPIC_ARN
-              value: {{ .SNS_TOPIC_ARN }}
-            - name: SNS_REGION
-              value: {{ .SNS_REGION }}
-          resources:
-            limits:
-              memory: {{ .RESOURCES_LIMITS_MEMORY }}
-              cpu: {{ .RESOURCES_LIMITS_CPU }}
-            requests:
-              memory: {{ .RESOURCES_REQUESTS_MEMORY }}
-              cpu: {{ .RESOURCES_REQUESTS_CPU }}
-          volumeMounts:
-            - name: metadata-volume
-              mountPath: /home/node/app/metadata-repo 
-      volumes:
-        - name: metadata-volume
-          persistentVolumeClaim:
-            claimName: blockchain-watcher-metadata-pvc

deploy/blockchain-watcher/workers/ethereum.yaml

@@ -0,0 +1,158 @@
+---          
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .NAME }}-eth
+  namespace: {{ .NAMESPACE }}
+  labels:
+    app: {{ .NAME }}-eth
+spec:
+  selector:
+    app: {{ .NAME }}-eth
+  ports:
+    - port: {{ .PORT }}
+      targetPort: {{ .PORT }}
+      name: {{ .NAME }}-eth
+      protocol: TCP
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: blockchain-watcher-eth-pvc
+  namespace: {{ .NAMESPACE }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Mi
+  storageClassName: gp2
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .NAME }}-eth-jobs
+  namespace: {{ .NAMESPACE }}
+data:
+  testnet-jobs.json: |-
+    [
+      {
+        "id": "poll-log-message-published-ethereum",
+        "chain": "ethereum",
+        "source": {
+          "action": "PollEvmLogs",
+          "config": {
+            "fromBlock": "10012499",
+            "blockBatchSize": 100,
+            "commitment": "latest",
+            "interval": 15000,
+            "addresses": ["0x706abc4E45D419950511e474C7B9Ed348A4a716c"],
+            "chain": "ethereum",
+            "topics": []
+          }
+        },
+        "handlers": [
+          {
+            "action": "HandleEvmLogs",
+            "target": "sns",
+            "mapper": "evmLogMessagePublishedMapper",
+            "config": {
+              "abi": "event LogMessagePublished(address indexed sender, uint64 sequence, uint32 nonce, bytes payload, uint8 consistencyLevel)",
+              "filter": {
+                "addresses": ["0x706abc4E45D419950511e474C7B9Ed348A4a716c"],
+                "topics": ["0x6eb224fb001ed210e379b335e35efe88672a8ce935d981a6896b27ffdf52a3b2"]
+              }
+            }
+          }
+        ]
+      }
+    ]
+  mainnet-jobs.json: |-
+    [
+      {
+        "id": "poll-log-message-published-ethereum",
+        "chain": "ethereum",
+        "source": {
+          "action": "PollEvmLogs",
+          "config": {
+            "blockBatchSize": 100,
+            "commitment": "latest",
+            "interval": 15000,
+            "addresses": ["0x98f3c9e6E3fAce36bAAd05FE09d375Ef1464288B"],
+            "chain": "ethereum",
+            "topics": []
+          }
+        },
+        "handlers": [
+          {
+            "action": "HandleEvmLogs",
+            "target": "sns",
+            "mapper": "evmLogMessagePublishedMapper",
+            "config": {
+              "abi": "event LogMessagePublished(address indexed sender, uint64 sequence, uint32 nonce, bytes payload, uint8 consistencyLevel)",
+              "filter": {
+                "addresses": ["0x98f3c9e6E3fAce36bAAd05FE09d375Ef1464288B"],
+                "topics": ["0x6eb224fb001ed210e379b335e35efe88672a8ce935d981a6896b27ffdf52a3b2"]
+              }
+            }
+          }
+        ]
+      }
+    ]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ .NAME }}-eth
+  namespace: {{ .NAMESPACE }}
+  labels:
+    app: {{ .NAME }}-eth
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "{{ .PORT }}"
+spec:
+  restartPolicy: Always
+  terminationGracePeriodSeconds: 30
+  serviceAccountName: event-watcher
+  containers:
+    - name: {{ .NAME }}
+      image: {{ .IMAGE_NAME }}
+      env:
+        - name: NODE_ENV
+          value: {{ .NODE_ENV }}
+        - name: PORT
+          value: "{{ .PORT }}"
+        - name: LOG_LEVEL
+          value: {{ .LOG_LEVEL }}
+        - name: BLOCKCHAIN_ENV
+          value: {{ .BLOCKCHAIN_ENV }}
+        - name: DRY_RUN_ENABLED
+          value: "{{ .DRY_RUN_ENABLED }}"
+        - name: SNS_TOPIC_ARN
+          value: {{ .SNS_TOPIC_ARN }}
+        - name: SNS_REGION
+          value: {{ .SNS_REGION }}
+        - name: JOBS_DIR
+          value: /home/node/app/jobs
+      resources:
+        limits:
+          memory: {{ .RESOURCES_LIMITS_MEMORY }}
+          cpu: {{ .RESOURCES_LIMITS_CPU }}
+        requests:
+          memory: {{ .RESOURCES_REQUESTS_MEMORY }}
+          cpu: {{ .RESOURCES_REQUESTS_CPU }}
+      volumeMounts:
+        - name: metadata-volume
+          mountPath: /home/node/app/metadata-repo 
+        - name: jobs-volume
+          mountPath: /home/node/app/jobs
+  volumes:
+    - name: metadata-volume
+      persistentVolumeClaim:
+        claimName: blockchain-watcher-eth-pvc
+    - name: jobs-volume
+      configMap:
+        name: {{ .NAME }}-eth-jobs
+        items:
+          - key: {{ .BLOCKCHAIN_ENV }}-jobs.json
+            path: jobs.json