inital commit

2023-11-28 17:15:23 +03:00
commit 030e2f85f5
15 changed files with 620 additions and 0 deletions
--- a/deployments/Makefile
+++ b/deployments/Makefile
@@ -0,0 +1,10 @@
+clean: stop
+	rm -rf data/kafka{1,2,3} && rm -rf data/zoo{1,2,3} && mkdir -p data/kafka{1,2,3} && mkdir -p data/zoo{1,2,3}
+start:
+	docker compose up -d
+stop:
+	docker compose down
+get_kafdrop:
+	wget -c https://github.com/obsidiandynamics/kafdrop/releases/download/4.0.1/kafdrop-4.0.1.jar
+kafdrop: get_kafdrop
+	java -jar kafdrop-4.0.1.jar --kafka.brokerConnect=127.0.0.1:29092,127.0.0.1:39092,127.0.0.1:49092
--- a/deployments/connect.properites
+++ b/deployments/connect.properites
@@ -0,0 +1,10 @@
+bootstrap.servers=127.0.0.1:29092,127.0.0.1:39092,127.0.0.1:49092
+group.id=myconnect
+
+key.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+
+offset.storage.topic=connector_offset
+offset.storage.file.filename=offset
+config.storage.topic=connector_config
+status.storage.topic=connector_status
--- a/deployments/docker-compose.yml
+++ b/deployments/docker-compose.yml
@@ -0,0 +1,133 @@
+---
+version: '3'
+services:
+  zookeeper-1:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_PEER_PORT: 2888
+      ZOOKEEPER_LEADER_PORT: 3888
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: "zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888"
+    ports:
+      - 22181:2181
+    volumes:
+      - ./data/zoo1/data:/var/lib/zookeeper/data
+      - ./data/zoo1/log:/var/lib/zookeeper/log
+
+  zookeeper-2:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_PEER_PORT: 2888
+      ZOOKEEPER_LEADER_PORT: 3888
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_SERVER_ID: 2
+      ZOOKEEPER_SERVERS: "zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888"
+    ports:
+      - 32181:2181
+    volumes:
+      - ./data/zoo2/data:/var/lib/zookeeper/data
+      - ./data/zoo2/log:/var/lib/zookeeper/log
+
+  zookeeper-3:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_PEER_PORT: 2888
+      ZOOKEEPER_LEADER_PORT: 3888
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_SERVER_ID: 3
+      ZOOKEEPER_SERVERS: "zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888"
+    ports:
+      - 42181:2181
+    volumes:
+      - ./data/zoo3/data:/var/lib/zookeeper/data
+      - ./data/zoo3/log:/var/lib/zookeeper/log
+  
+  kafka-1:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    ports:
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_BROKER_RACK: Docker
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181/kafka
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_JMX_PORT: 39999
+    volumes:
+      - ./data/kafka1:/var/lib/kafka/data
+
+  kafka-2:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    ports:
+      - 39092:39092
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_BROKER_RACK: Docker
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181/kafka
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://localhost:39092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_JMX_PORT: 39999
+    volumes:
+      - ./data/kafka2:/var/lib/kafka/data
+
+  kafka-3:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    ports:
+      - 49092:49092
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_BROKER_RACK: Docker
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181/kafka
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092,PLAINTEXT_HOST://localhost:49092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_JMX_PORT: 39999
+    volumes:
+      - ./data/kafka3:/var/lib/kafka/data
+
+
+  kafka-exporter:
+    image: danielqsj/kafka-exporter
+    depends_on:
+      - kafka-1
+      - kafka-2
+      - kafka-3
+    command:
+       - --kafka.server=kafka-1:9092
+       - --kafka.server=kafka-2:9092
+       - --kafka.server=kafka-3:9092
+    ports:
+      - 9308:9308
+
+  # kafdrop:
+  #   image: obsidiandynamics/kafdrop
+  #   depends_on:
+  #     - kafka-1
+  #     - kafka-2
+  #     - kafka-3
+  #   environment:
+  #     KAFKA_BROKERCONNECT: kafka-1:9092,kafka-2:9092,kafka-3:9092
+  #     # CMD_ARGS: --schemaregistry.connect=http://schema-registry:8081
+  #   ports:
+  #     - 9000:9000
--- a/deployments/mongodb.json
+++ b/deployments/mongodb.json
@@ -0,0 +1,9 @@
+{"name": "mongo-source",
+ "config": {
+    "connector.class":"com.mongodb.kafka.connect.MongoSourceConnector",
+    "connection.uri":"mongodb://localhost:27017",
+    "database":"strava",
+    "collection":"workout",
+	"copy.existing": true
+    }
+}