update(bi): kafka

projects/ai/manifests/kafka.yml

@@ -0,0 +1,111 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: bi-cluster
+  namespace: bibot
+spec:
+  kafka:
+    version: 3.7.0
+    replicas: 1
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      offsets.topic.replication.factor: 1
+      transaction.state.log.replication.factor: 1
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 1
+      min.insync.replicas: 1
+      inter.broker.protocol.version: "3.7"
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 100Gi
+        deleteClaim: false
+  zookeeper:
+    replicas: 1
+    storage:
+      type: persistent-claim
+      size: 100Gi
+      deleteClaim: false
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: bi-input
+  namespace: bibot
+  labels:
+    strimzi.io/cluster: bi-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: bi-output
+  namespace: bibot
+  labels:
+    strimzi.io/cluster: bi-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: agent-registry
+  namespace: bibot
+  labels:
+    strimzi.io/cluster: bi-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: agent-main-input
+  namespace: bibot
+  labels:
+    strimzi.io/cluster: bi-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: discord-input
+  namespace: bibot
+  labels:
+    strimzi.io/cluster: bi-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824

projects/ai/values/bibot.yml

@@ -1,51 +1,56 @@
 controllers:
+  # main agent
   main:
     containers:
       main:
         image:
-          repository: reg.dc/bibot
+          repository: reg.dc/bi
           tag: latest
           pullPolicy: Always
-
         env:
-          OPENAI_API_KEY: fake
           BIBOT_CONFIG: /config/bibot.yml
+          BIBOT_KAFKA__BROKER: kafka://bi-cluster-kafka-bootstrap:9092
           PHOENIX_COLLECTOR_ENDPOINT: http://phoenix.phoenix.svc.cluster.local:6006
           PHOENIX_PROJECT_NAME: bi
-          OMBI_API_TOKEN:
+        command: ["python3"]
-            valueFrom:
+        args: ["/app/bi/agents/main/app.py", "worker", "-l", "info"]
-              secretKeyRef:
+
-                name: bibot
+  controller:
-                key: ombi-api-key
-          OPENWEATHERMAP_API_KEY:
-            valueFrom:
-              secretKeyRef:
-                name: bibot
-                key: OPENWEATHERMAP_API_KEY
-  bot:
     containers:
       main:
         image:
-          repository: reg.dc/bibot
+          repository: reg.dc/bi
+          tag: latest
+          pullPolicy: Always
+        env:
+          BIBOT_CONFIG: /config/bibot.yml
+          BIBOT_KAFKA__BROKER: kafka://bi-cluster-kafka-bootstrap:9092
+        command: ["python3"]
+        args: ["/app/bi/controller.py", "worker", "-l", "info"]
+
+  discord:
+    containers:
+      main:
+        image:
+          repository: reg.dc/bi
           tag: latest
           pullPolicy: Always
     
         command: ["python3"]
-        args: ["-u", "discord_bot/main.py"]
+        args: ["/app/bi/connectors/discord/app.py", "worker", "-l", "info"]
 
         env:
+          BIBOT_KAFKA__BROKER: bi-cluster-kafka-bootstrap:9092
           OPENAI_API_KEY: fake
-          BIBOT_DISCORD_TOKEN:
+          BIBOT_DISCORD__TOKEN:
             valueFrom:
               secretKeyRef:
                 name: bibot
                 key: discord-token
           ## Prod:
-          BIBOT_DISCORD_CHANNELS: "1216440541064200192"
+          BIBOT_DISCORD__CHANNELS: "1216440541064200192"
-
           # Dev:
           # BIBOT_DISCORD_CHANNELS: "1217418069693960223"
-          BIBOT_API_URL: "http://{{ .Release.Name }}.{{ .Release.Namespace }}.svc.cluster.local:8000/v1"
         probes:
           liveness: 
             enabled: false
@@ -70,10 +75,10 @@ persistence:
     type: persistentVolumeClaim
     accessMode: ReadWriteOnce
 
-service:
+# service:
-  main:
+#   main:
-    controller: main
+#     controller: main
-    ports:
+#     ports:
-      http:
+#       http:
-        port: 8000
+#         port: 8000
-    type: ClusterIP
+#     type: ClusterIP