From 87bcb72b76d19a61df789f4b31c13c3608d025a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Denquin?= Date: Thu, 19 Dec 2024 11:45:13 +0100 Subject: [PATCH] chore(scalability): Add webhook worker (#439) --- docker-compose.dev.yml | 48 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 54 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 97e7725..4a75fa6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -114,6 +114,7 @@ services: - SIDEKIQ_PDFS=true - SIDEKIQ_BILLING=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -168,6 +169,7 @@ services: - SIDEKIQ_PDFS=true - SIDEKIQ_BILLING=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -209,6 +211,7 @@ services: - SIDEKIQ_EVENTS=true - SIDEKIQ_BILLING=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -249,6 +252,7 @@ services: - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - SIDEKIQ_PDFS=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -290,6 +294,7 @@ services: - SIDEKIQ_PDFS=true - SIDEKIQ_BILLING=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -331,6 +336,49 @@ services: - SIDEKIQ_PDFS=true - SIDEKIQ_BILLING=true - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true + - LAGO_CLICKHOUSE_ENABLED=true + - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true + - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 + - LAGO_KAFKA_RAW_EVENTS_TOPIC=events-raw + - LAGO_KAFKA_ENRICHED_EVENTS_TOPIC=events_enriched + - LAGO_KAFKA_CLICKHOUSE_CONSUMER_GROUP=clickhouse + - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} + - LAGO_LICENSE=${LAGO_LICENSE:-} + + api-webhook-worker: + image: api_dev + container_name: lago_api_webhook_worker_dev + depends_on: + - api + restart: unless-stopped + command: bash -c "bundle install && ./scripts/start.webhook.worker.sh" + build: + context: ./api + dockerfile: $LAGO_PATH/api/Dockerfile.dev + volumes: + - $LAGO_PATH/api:/app:delegated + environment: + - LAGO_API_URL=https://api.lago.dev + - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - REDIS_URL=redis://redis:6379 + - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} + - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} + - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${ENCRYPTION_DETERMINISTIC_KEY:-your-encrpytion-deterministic-key} + - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${ENCRYPTION_KEY_DERIVATION_SALT:-your-encrpytion-derivation-salt} + - LAGO_USE_AWS_S3=${LAGO_USE_AWS_S3:-false} + - LAGO_FRONT_URL=${LAGO_FRONT_URL:-https://app.lago.dev} + - LAGO_PARALLEL_THREADS_COUNT=${LAGO_PARALLEL_THREADS_COUNT:-4} + - LAGO_PDF_URL=${LAGO_PDF_URL:-http://pdf:3000} + - LAGO_REDIS_CACHE_URL=redis://redis:6379 + - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY:-} + - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} + - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} + - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} + - SIDEKIQ_PDFS=true + - SIDEKIQ_BILLING=true + - SIDEKIQ_CLOCK=true + - SIDEKIQ_WEBHOOK=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 diff --git a/docker-compose.yml b/docker-compose.yml index 9f40f0c..29630b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: # - SIDEKIQ_PDFS=true # - SIDEKIQ_BILLING=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true volumes: - lago_storage_data:/app/storage # If using GCS, you need to put the credentials keyfile here @@ -175,6 +176,7 @@ services: # - SIDEKIQ_PDFS=true # - SIDEKIQ_BILLING=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true volumes: - lago_storage_data:/app/storage @@ -220,6 +222,7 @@ services: # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} # - SIDEKIQ_EVENTS=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true # - LAGO_LICENSE=${LAGO_LICENSE:-} # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices pdf creation. @@ -264,6 +267,7 @@ services: # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} # - SIDEKIQ_PDFS=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true # - LAGO_LICENSE=${LAGO_LICENSE:-} # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices creation. @@ -309,10 +313,10 @@ services: # - SIDEKIQ_PDFS=true # - SIDEKIQ_BILLING=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true # - LAGO_LICENSE=${LAGO_LICENSE:-} - # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices creation. - # It is recommended if you have a high usage of invoices being created to not impact the other Sidekiq Jobs. + # You can uncomment this if you want to use a dedicated Sidekiq worker for the clock jobs. #api-clock-worker: # container_name: lago-clock-worker # image: getlago/api:v1.17.2 @@ -354,6 +358,52 @@ services: # - SIDEKIQ_PDFS=true # - SIDEKIQ_BILLING=true # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true + # - LAGO_LICENSE=${LAGO_LICENSE:-} + + # You can uncomment this if you want to use a dedicated Sidekiq worker for the webhook jobs. + #api-webhook-worker: + # container_name: lago-webhook-worker + # image: getlago/api:v1.17.2 + # restart: unless-stopped + # depends_on: + # api: + # condition: service_healthy + # command: ["./scripts/start.webhook.worker.sh"] + # environment: + # - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} + # - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} + # - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379} + # - REDIS_PASSWORD=${REDIS_PASSWORD} + # - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} + # - RAILS_ENV=production + # - RAILS_LOG_TO_STDOUT=${LAGO_RAILS_STDOUT:-true} + # - SENTRY_DSN=${SENTRY_DSN} + # - LAGO_RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded + # - RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded + # - LAGO_ENCRYPTION_PRIMARY_KEY=${LAGO_ENCRYPTION_PRIMARY_KEY:-your-encryption-primary-key} + # - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${LAGO_ENCRYPTION_DETERMINISTIC_KEY:-your-encryption-deterministic-key} + # - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${LAGO_ENCRYPTION_KEY_DERIVATION_SALT:-your-encryption-derivation-salt} + # - LAGO_FRONT_URL=${LAGO_FRONT_URL:-http://localhost} + # - LAGO_USE_AWS_S3=${LAGO_USE_AWS_S3:-false} + # - LAGO_AWS_S3_ACCESS_KEY_ID=${LAGO_AWS_S3_ACCESS_KEY_ID:-azerty123456} + # - LAGO_AWS_S3_SECRET_ACCESS_KEY=${LAGO_AWS_S3_SECRET_ACCESS_KEY:-azerty123456} + # - LAGO_AWS_S3_REGION=${LAGO_AWS_S3_REGION:-us-east-1} + # - LAGO_AWS_S3_BUCKET=${LAGO_AWS_S3_BUCKET:-bucket} + # - LAGO_AWS_S3_ENDPOINT=${LAGO_AWS_S3_ENDPOINT} + # - LAGO_USE_GCS=${LAGO_USE_GCS:-false} + # - LAGO_GCS_PROJECT=${LAGO_GCS_PROJECT:-} + # - LAGO_GCS_BUCKET=${LAGO_GCS_BUCKET:-} + # - LAGO_PDF_URL=${LAGO_PDF_URL:-http://pdf:3000} + # - LAGO_REDIS_CACHE_URL=redis://${LAGO_REDIS_CACHE_HOST:-redis}:${LAGO_REDIS_CACHE_PORT:-6379} + # - LAGO_REDIS_CACHE_PASSWORD=${LAGO_REDIS_CACHE_PASSWORD} + # - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT} + # - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} + # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} + # - SIDEKIQ_PDFS=true + # - SIDEKIQ_BILLING=true + # - SIDEKIQ_CLOCK=true + # - SIDEKIQ_WEBHOOK=true # - LAGO_LICENSE=${LAGO_LICENSE:-} api-clock: