- 第一部分:简介
- 第二部分:代理商和团队
我是如何来到这一生的?
不久前,我不得不在一个高负载项目的后端工作,在该项目中,我需要组织大量包含复杂计算和对第三方服务的请求的后台任务的常规执行。该项目是异步的,在我接触它之前,它有一个简单的机制可以快速启动任务:一个循环,用于检查当前时间并通过收集启动协程程序组-这种方法在直到数十个这样的协程程序被接受之前是可以接受的。他们的人数超过了2000,所以我不得不考虑与经纪人,几个工人等等组织一个正常的任务队列。
, , , , . , , , (. group). issue , , . , , … , . , 2-3 http- , 4 tcp , 2 — … . - aiohttp .
, ! celery, , Ask Solem, Faust, robinhood. Faust Kafka Streams Kafka , rocksdb, — , .
, celery faust : , , . , , faust — .
?
, , Faust. - , , alphavantage.co. , (sink, , ), (cron) , cli- faust ( click), , datadog ( ) , - . mongodb motor .
P.S. , , , - , - :
, , , :
- overview ( .. , , cash flow — ) —
- ( ) —
- —
- —
, : horton
, , — docker-compose kafka ( zookeeper — ), kafdrop ( ), mongodb. [docker-compose.yml](https://github.com/Egnod/horton/blob/562fa5ec14df952cd74760acf76e141707d2ef58/docker-compose.yml) :
version: '3'
services:
db:
container_name: horton-mongodb-local
image: mongo:4.2-bionic
command: mongod --port 20017
restart: always
ports:
- 20017:20017
environment:
- MONGO_INITDB_DATABASE=horton
- MONGO_INITDB_ROOT_USERNAME=admin
- MONGO_INITDB_ROOT_PASSWORD=admin_password
kafka-service:
container_name: horton-kafka-local
image: obsidiandynamics/kafka
restart: always
ports:
- "2181:2181"
- "9092:9092"
environment:
KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092"
KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-service:29092,EXTERNAL://localhost:9092"
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000"
KAFKA_RESTART_ATTEMPTS: "10"
KAFKA_RESTART_DELAY: "5"
ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0"
kafdrop:
container_name: horton-kafdrop-local
image: 'obsidiandynamics/kafdrop:latest'
restart: always
ports:
- '9000:9000'
environment:
KAFKA_BROKERCONNECT: kafka-service:29092
depends_on:
- kafka-service
. kafka listener': (internal) , (external) , . 2181 — zookeeper'. , , .
:
horton
├── docker-compose.yml
└── horton
├── agents.py *
├── alphavantage.py *
├── app.py *
├── config.py
├── database
│ ├── connect.py
│ ├── cruds
│ │ ├── base.py
│ │ ├── __init__.py
│ │ └── security.py *
│ └── __init__.py
├── __init__.py
├── records.py *
└── tasks.py *
, , .
. , mongodb. , , .
, virtualenv (, venv ):
pip3 install poetry ( )
poetry install
config.yml — . alphavantage. config.py — . , , — sitri.
?
, , — , .
, :
- alphavantage aiohttp .
- , .