This repository has been archived by the owner on Mar 1, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
/
twitter-compose.yml
128 lines (123 loc) · 3.78 KB
/
twitter-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
version: "3.2"
services:
# KAFKA
zookeeper:
image: "wurstmeister/zookeeper"
ports:
- "2181:2181"
my-kafka:
image: "wurstmeister/kafka:0.10.2.0"
ports:
- "9092:9092"
depends_on:
- "zookeeper"
environment:
KAFKA_CREATE_TOPICS: >
twitter.scraped.user_names:4:1,twitter.scraped.users:4:1,twitter.scraped.posts:4:1
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_LISTENERS: "PLAINTEXT://:9092"
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://my-kafka:9092"
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
# POSTGRES
postgres:
image: "debezium/postgres:11"
ports:
- "5432:5432"
environment:
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "12345678"
POSTGRES_DB: "smag-twitter"
connect:
image: "debezium/connect:0.10"
ports:
- "8083:8083"
environment:
BOOTSTRAP_SERVERS: "my-kafka:9092"
GROUP_ID: "1"
CONFIG_STORAGE_TOPIC: "my_connect_configs"
OFFSET_STORAGE_TOPIC: "my_connect_offsets"
STATUS_STORAGE_TOPIC: "my_connect_statuses"
setup-connect:
build: "twitter/debezium"
# TWITTER SCRAPER
twitter_scraper_users:
build: "twitter/scraper/"
image: "smag-twitter_scraper"
command: ["-m", "twitterscraper.users_scraper"]
depends_on:
- "my-kafka"
environment:
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_FETCH_TOPIC: "twitter.scraped.user_names"
KAFKA_INSERT_TOPIC: "twitter.scraped.users"
twitter_scraper_followers:
build: "twitter/scraper/"
image: "smag-twitter_scraper"
command: ["-m", "twitterscraper.followers_scraper"]
depends_on:
- "my-kafka"
environment:
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_FETCH_TOPIC: "twitter.scraped.user_names"
KAFKA_INSERT_TOPIC: "twitter.scraped.users"
twitter_scraper_following:
build: "twitter/scraper/"
image: "smag-twitter_scraper"
command: ["-m", "twitterscraper.following_scraper"]
depends_on:
- "my-kafka"
environment:
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_FETCH_TOPIC: "twitter.scraped.user_names"
KAFKA_INSERT_TOPIC: "twitter.scraped.users"
twitter_scraper_posts:
build: "twitter/scraper/"
image: "smag-twitter_scraper"
command: ["-m", "twitterscraper.posts_scraper"]
depends_on:
- "my-kafka"
environment:
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_FETCH_TOPIC: "twitter.scraped.user_names"
KAFKA_INSERT_TOPIC: "twitter.scraped.posts"
# TWITTER INSERTER
twitter_inserter_users:
build:
context: "."
dockerfile: "twitter/inserter/users/Dockerfile"
image: "smag-twitter_inserter_users"
environment:
POSTGRES_HOST: "postgres"
POSTGRES_PASSWORD: "12345678"
POSTGRES_DB_NAME: "smag-twitter"
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_GROUPID: "twitter.inserter.users"
KAFKA_INFO_TOPIC: "twitter.scraped.users"
KAFKA_NAME_TOPIC: "twitter.scraped.user_names"
twitter_inserter_posts:
build:
context: "."
dockerfile: "twitter/inserter/posts/Dockerfile"
image: "smag-twitter_inserter_posts"
environment:
POSTGRES_HOST: "postgres"
POSTGRES_PASSWORD: "12345678"
POSTGRES_DB_NAME: "smag-twitter"
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_GROUPID: "twitter_inserter_posts"
KAFKA_INFO_TOPIC: "twitter.scraped.posts"
KAFKA_NAME_TOPIC: "twitter.scraped.user_names"
# TWITTER FILTER
twitter_filter_user_names:
build:
context: "."
dockerfile: "twitter/filter/user_names/Dockerfile"
depends_on:
- "postgres"
- "connect"
environment:
KAFKA_ADDRESS: "my-kafka:9092"
KAFKA_GROUPID: "twitter.filter.user_names"
KAFKA_CHANGE_TOPIC: "postgres.public.twitter_users"
KAFKA_NAME_TOPIC: "twitter.scraped.user_names"