Sebastian Krüger
6df2de5bb2
Added scrapy-ui service to scrapy stack: - Image: ghcr.io/valknarness/scrapy-ui:latest - Path-based routing: https://scrapy.pivoine.art/ui - HTTP Basic Auth protection (same credentials as scrapyd) - StripPrefix middleware to remove /ui from requests - Auto-connects to scrapyd using SCRAPYD_URL, USERNAME, PASSWORD - Watchtower auto-updates enabled Traefik configuration: - Middleware chain: auth -> stripprefix -> compress - Routes /ui prefix to port 3000 - Shares authentication with main scrapyd interface 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
96 lines
4.8 KiB
YAML
96 lines
4.8 KiB
YAML
services:
|
|
scrapyd:
|
|
image: ${SCRAPY_SCRAPYD_IMAGE:-vimagick/scrapyd}
|
|
container_name: ${SCRAPY_COMPOSE_PROJECT_NAME}_scrapyd
|
|
restart: unless-stopped
|
|
volumes:
|
|
- scrapyd_data:/var/lib/scrapyd
|
|
- /usr/local/lib/python3.9/dist-packages
|
|
environment:
|
|
TZ: ${TIMEZONE:-Europe/Berlin}
|
|
networks:
|
|
- compose_network
|
|
labels:
|
|
- 'traefik.enable=${SCRAPY_TRAEFIK_ENABLED}'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-redirect-web-secure.redirectscheme.scheme=https'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web.middlewares=${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-redirect-web-secure'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web.rule=Host(`${SCRAPY_TRAEFIK_HOST}`)'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web.entrypoints=web'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure.rule=Host(`${SCRAPY_TRAEFIK_HOST}`)'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure.tls.certresolver=resolver'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure.entrypoints=web-secure'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-auth.basicauth.users=${SCRAPY_AUTH_USERS}'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure-compress.compress=true'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure.middlewares=${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-auth,${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure-compress'
|
|
- 'traefik.http.services.${SCRAPY_COMPOSE_PROJECT_NAME}-scrapyd-web-secure.loadbalancer.server.port=6800'
|
|
- 'traefik.docker.network=${NETWORK_NAME}'
|
|
- 'com.centurylinklabs.watchtower.enable=${WATCHTOWER_LABEL_ENABLE}'
|
|
|
|
scrapy:
|
|
image: ${SCRAPY_IMAGE:-vimagick/scrapyd}
|
|
container_name: ${SCRAPY_COMPOSE_PROJECT_NAME}_scrapy
|
|
command: bash
|
|
volumes:
|
|
- scrapy_code:/code
|
|
working_dir: /code
|
|
restart: unless-stopped
|
|
environment:
|
|
TZ: ${TIMEZONE:-Europe/Berlin}
|
|
networks:
|
|
- compose_network
|
|
|
|
scrapyrt:
|
|
image: ${SCRAPY_SCRAPYRT_IMAGE:-vimagick/scrapyd}
|
|
container_name: ${SCRAPY_COMPOSE_PROJECT_NAME}_scrapyrt
|
|
command: scrapyrt -i 0.0.0.0 -p 9080
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${SCRAPY_SCRAPYRT_PORT:-9080}:9080"
|
|
volumes:
|
|
- scrapy_code:/code
|
|
working_dir: /code
|
|
environment:
|
|
TZ: ${TIMEZONE:-Europe/Berlin}
|
|
networks:
|
|
- compose_network
|
|
|
|
scrapy_ui:
|
|
image: ${SCRAPY_UI_IMAGE:-ghcr.io/valknarness/scrapy-ui:latest}
|
|
container_name: ${SCRAPY_COMPOSE_PROJECT_NAME}_ui
|
|
restart: unless-stopped
|
|
environment:
|
|
TZ: ${TIMEZONE:-Europe/Berlin}
|
|
NODE_ENV: production
|
|
SCRAPYD_URL: ${SCRAPY_UI_SCRAPYD_URL:-https://scrapy.pivoine.art}
|
|
SCRAPYD_USERNAME: ${SCRAPY_UI_SCRAPYD_USERNAME}
|
|
SCRAPYD_PASSWORD: ${SCRAPY_UI_SCRAPYD_PASSWORD}
|
|
networks:
|
|
- compose_network
|
|
labels:
|
|
- 'traefik.enable=${SCRAPY_TRAEFIK_ENABLED}'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-redirect-web-secure.redirectscheme.scheme=https'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web.middlewares=${SCRAPY_COMPOSE_PROJECT_NAME}-ui-redirect-web-secure'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web.rule=Host(`${SCRAPY_TRAEFIK_HOST}`) && PathPrefix(`/ui`)'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web.entrypoints=web'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure.rule=Host(`${SCRAPY_TRAEFIK_HOST}`) && PathPrefix(`/ui`)'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure.tls.certresolver=resolver'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure.entrypoints=web-secure'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-stripprefix.stripprefix.prefixes=/ui'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-auth.basicauth.users=${SCRAPY_AUTH_USERS}'
|
|
- 'traefik.http.middlewares.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure-compress.compress=true'
|
|
- 'traefik.http.routers.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure.middlewares=${SCRAPY_COMPOSE_PROJECT_NAME}-ui-auth,${SCRAPY_COMPOSE_PROJECT_NAME}-ui-stripprefix,${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure-compress'
|
|
- 'traefik.http.services.${SCRAPY_COMPOSE_PROJECT_NAME}-ui-web-secure.loadbalancer.server.port=3000'
|
|
- 'traefik.docker.network=${NETWORK_NAME}'
|
|
- 'com.centurylinklabs.watchtower.enable=${WATCHTOWER_LABEL_ENABLE}'
|
|
|
|
volumes:
|
|
scrapyd_data:
|
|
name: ${SCRAPY_COMPOSE_PROJECT_NAME}_scrapyd_data
|
|
scrapy_code:
|
|
name: ${SCRAPY_COMPOSE_PROJECT_NAME}_scrapy_code
|
|
|
|
networks:
|
|
compose_network:
|
|
name: ${NETWORK_NAME}
|
|
external: true
|