diff --git a/.drone.yml b/.drone.yml index eec79f2..9b28198 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,39 +2,59 @@ kind: pipeline name: default steps: + # - name: docker-build + # image: plugins/docker + # settings: + # username: + # from_secret: DOCKER_USER + # password: + # from_secret: DOCKER_PASS + # repo: mhackspace-python + # dockerfile: ./compose/django/Dockerfile + # tags: latest + # repo: docker.digitaloctave.com/do-registry + # registry: docker.digitaloctave.com + - name: tests - image: python:3.6 + image: olymk2/python-built:3.6-alpine environment: PYTHONUSERBASE: /drone/src/cache/packages POSTGRES_USER: mhackspace USE_DOCKER: yes DJANGO_SETTINGS_MODULE: config.settings.test - BUCKET_URL: http://minio:9000 commands: - - cp -n env.example .env + - apk add git build-base gcc python3-dev postgresql-dev musl-dev jpeg-dev zlib-dev openjpeg-dev tiff-dev libffi-dev freetype-dev libev-dev lcms2-dev tk-dev tcl-dev harfbuzz-dev fribidi-dev libxslt-dev + - cp env.example .env - mkdir -p ./cache/packages ./cache/pip - - pip install --user --cache-dir ./cache/pip -r ./requirements/test.txt + - pip install -r ./requirements/test.txt - python manage.py compilescss - python manage.py collectstatic --no-input - pytest -v # - python manage.py test mhackspace --keepdb --verbosity 2 + + - name: publish-stage pull: True image: appleboy/drone-ssh - host: stage.maidstone-hackspace.org.uk - user: root - secrets: [ ssh_key ] - port: 22 - command_timeout: 480 - script: - - cd /var/www/maidstone-hackspace-website - - git pull - - docker-compose -fstage.yml build - - docker-compose -fstage.yml run --rm django python manage.py compilescss - - docker-compose -fstage.yml run --rm django python manage.py collectstatic - - docker-compose -fstage.yml down --remove-orphans - - docker-compose -fstage.yml up -d + settings: + host: stage.maidstone-hackspace.org.uk + + username: + from_secret: ssh_user + key: + from_secret: ssh_key + port: 22 + command_timeout: 30m + script: + - cd /var/www/maidstone-hackspace-website + - git pull + - docker-compose -fstage.yml build + - docker-compose -fstage.yml run --rm django python manage.py compilescss + - docker-compose -fstage.yml run --rm django python manage.py collectstatic + - docker-compose -fstage.yml down --remove-orphans + - docker-compose -fstage.yml up -d + - echo 'Finished deploy' - name: publish-live pull: True @@ -45,7 +65,7 @@ steps: target: ssh_key host: maidstone-hackspace.org.uk port: 22 - command_timeout: 480 + command_timeout: 5m script: - echo 'to be added' when: @@ -61,7 +81,7 @@ services: - npm --version - npm install - - name: minio + - name: bucket image: minio/minio environment: MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile index 7bc1c20..aec74eb 100644 --- a/compose/django/Dockerfile +++ b/compose/django/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-alpine +FROM olymk2/python-built:3.6-alpine ENV PYTHONUNBUFFERED 1 @@ -10,7 +10,7 @@ RUN apk add --no-cache git build-base gcc \ COPY ./requirements /requirements -RUN pip install --cache-dir ./cache/pip -r /requirements/local.txt \ +RUN pip install --no-cache-dir -r /requirements/local.txt \ && addgroup -g 1000 -S django \ && adduser -u 1000 -S django -G django diff --git a/compose/django/Dockerfile-dev b/compose/django/Dockerfile-dev deleted file mode 100644 index 7bc1c20..0000000 --- a/compose/django/Dockerfile-dev +++ /dev/null @@ -1,33 +0,0 @@ -FROM python:3.6-alpine - -ENV PYTHONUNBUFFERED 1 - -RUN apk add --no-cache git build-base gcc \ - python3-dev postgresql-dev musl-dev \ - jpeg-dev zlib-dev openjpeg-dev tiff-dev libffi-dev \ - freetype-dev libev-dev lcms2-dev tk-dev tcl-dev \ - harfbuzz-dev fribidi-dev libxslt-dev - -COPY ./requirements /requirements - -RUN pip install --cache-dir ./cache/pip -r /requirements/local.txt \ - && addgroup -g 1000 -S django \ - && adduser -u 1000 -S django -G django - -COPY . /app - -RUN mkdir -p /data/sockets -# && chown -R django /app - -COPY ./compose/django/bjoern.py /bjoern.py -COPY ./compose/django/entrypoint.sh /entrypoint.sh - - -RUN sed -i 's/\r//' /entrypoint.sh \ - && chmod +x /entrypoint.sh \ - && chown django /entrypoint.sh \ - && chown django /data/sockets - -WORKDIR /app - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/compose/django/entrypoint.sh b/compose/django/entrypoint.sh index e13adb7..633a236 100644 --- a/compose/django/entrypoint.sh +++ b/compose/django/entrypoint.sh @@ -15,8 +15,6 @@ fi export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:5432/$POSTGRES_USER -export CELERY_BROKER_URL=$REDIS_URL/0 - function postgres_ready(){ python << END diff --git a/config/settings/common.py b/config/settings/common.py index 95f67b8..8f35157 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -626,6 +626,7 @@ HUEY = { 'store_errors': True, # Store error info if task throws exception. 'blocking': False, # Poll the queue rather than do blocking pop. 'backend_class': 'huey.RedisHuey', # Use path to redis huey by default, + 'immediate': False, 'connection': { 'host': 'redis', 'port': 6379, @@ -635,7 +636,6 @@ HUEY = { # huey-specific connection parameters. 'read_timeout': 1, # If not polling (blocking pop), use timeout. - 'max_errors': 1000, # Only store the 1000 most recent errors. 'url': None, # Allow Redis config via a DSN. }, 'consumer': { diff --git a/live.yml b/live.yml index 112e515..9ea6dd4 100644 --- a/live.yml +++ b/live.yml @@ -24,6 +24,8 @@ services: user: django depends_on: - postgres + - directory + - redis command: /live-gunicorn-mhackspace.sh env_file: .env volumes: @@ -39,10 +41,11 @@ services: huey: build: context: . - dockerfile: ./compose/django/Dockerfile-dev + dockerfile: ./compose/django/Dockerfile user: django depends_on: - postgres + - directory - redis command: python manage.py run_huey env_file: .env diff --git a/local.yml b/local.yml index 32a8637..c22311d 100644 --- a/local.yml +++ b/local.yml @@ -29,7 +29,7 @@ services: huey: build: context: . - dockerfile: ./compose/django/Dockerfile-dev + dockerfile: ./compose/django/Dockerfile user: django depends_on: - postgres @@ -43,7 +43,7 @@ services: django: build: context: . - dockerfile: ./compose/django/Dockerfile-dev + dockerfile: ./compose/django/Dockerfile user: django depends_on: - postgres @@ -57,7 +57,7 @@ services: django_bjoren: build: context: . - dockerfile: ./compose/django/Dockerfile-dev + dockerfile: ./compose/django/Dockerfile user: django depends_on: - postgres diff --git a/mhackspace/ldapsync/tasks.py b/mhackspace/ldapsync/tasks.py index d804f20..b77fce4 100644 --- a/mhackspace/ldapsync/tasks.py +++ b/mhackspace/ldapsync/tasks.py @@ -9,7 +9,7 @@ import json server = Server(settings.LDAP_SERVER) conn = Connection( server, - 'cn=admin, dc=maidstone-hackspace, dc=org, dc=uk', + f'cn=admin, {settings.LDAP_ROOT}', settings.LDAP_PASSWORD, auto_bind=True) diff --git a/mhackspace/users/admin.py b/mhackspace/users/admin.py index 33833c4..5dd4e42 100644 --- a/mhackspace/users/admin.py +++ b/mhackspace/users/admin.py @@ -61,7 +61,7 @@ class MembershipAdmin(ModelAdmin): return my_urls + urls def refresh_payments(self, request): - update_users_memebership_status.apply_async() + update_users_memebership_status() # for user in update_subscriptions(provider_name='gocardless'): # continue self.message_user(request, 'Successfully triggered user payment refresh') diff --git a/requirements/base.txt b/requirements/base.txt index e06ed59..4678703 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,16 +2,16 @@ # like Pillow and psycopg2 # See http://bitly.com/wheel-building-fails-CPython-35 # Verified bug on Python 3.5.1 -wheel==0.32.3 +wheel==0.33.1 # Bleeding edge Django -django==2.1.7 +django==2.2.1 django-dynamic-filenames==1.1.4 # Configuration django-environ==0.4.5 whitenoise==4.1.2 # Static and Media Storage # ------------------------------------------------ -boto3==1.9.96 +boto3==1.9.142 django-storages==1.7.1 # django-storages-redux==1.3.2 @@ -20,29 +20,29 @@ django-storages==1.7.1 django-braces==1.13.0 django-crispy-forms==1.7.2 -django-extensions==2.1.5 -Werkzeug==0.14.1 +django-extensions==2.1.6 +Werkzeug==0.15.2 # Models django-stdimage==4.0.1 django-model-utils==3.1.2 # Images -Pillow==5.4.1 +Pillow==6.0.0 # For user registration, either via email or social # Well-built with regular release cycles! -django-allauth==0.38.0 +django-allauth==0.39.1 # Python-PostgreSQL Database Adapter -psycopg2==2.7.7 +psycopg2==2.8.2 # Unicode slugification awesome-slugify==1.6.5 # Time zones support -pytz==2018.9 +pytz==2019.1 # Redis support django-redis==4.10.0 @@ -54,45 +54,45 @@ django-compressor==2.2 #fix for use with s3 buckets merged in master, so next release we can remove this #django-sass-processor==0.5.7 git+https://github.com/jrief/django-sass-processor.git -libsass==0.17.0 -lxml==4.3.1 +libsass==0.18.0 +lxml==4.3.3 # WSGI Handler # ------------------------------------------------ gevent==1.4.0 gunicorn==19.9.0 #https://github.com/jonashaag/bjoern -bjoern +bjoern==2.2.3 # Your custom requirements go here -mock==2.0.0 +mock==3.0.4 gocardless_pro==1.9.0 -braintree==3.51.0 +braintree==3.53.0 django-autofixture==0.12.1 wiki==0.4.4 -djangorestframework==3.9.1 +djangorestframework==3.9.3 djangorestframework-jwt==1.11.0 django-filter==2.1.0 coreapi==2.3.3 # api libraries end -martor==1.3.8 +martor==1.4.0 -django-spirit==0.7.0 -django-djconfig==0.9.0 +django-spirit==0.7.1 +django-djconfig==0.10.0 django-haystack==2.8.1 django-xforwardedfor-middleware==2.0 # Application queue -huey[backends] +huey[backends]==2.0.1 argon2-cffi==19.1.0 -django-cors-headers==2.4.0 +django-cors-headers==2.5.3 python-magic==0.4.15 -ldap3==2.5.2 +ldap3==2.6 bcrypt==3.1.6 python-twitter==3.5 feedparser==5.2.1 diff --git a/requirements/test.txt b/requirements/test.txt index c8c5c8a..709a256 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -2,15 +2,15 @@ -r base.txt # linting -flake8==3.7.5 # pyup: != 2.6.0 +flake8==3.7.7 # pyup: != 2.6.0 -coverage==4.5.2 +coverage==4.5.3 factory-boy==2.11.1 django-coverage-plugin==1.6.0 django-test-plus==1.1.1 -# pytest -pytest-django==3.4.7 +pytest==4.4.1 +pytest-django==3.4.8 pytest-sugar==0.9.2 model_mommy==1.6.0 diff --git a/stage.yml b/stage.yml index 20f2a15..09c415d 100644 --- a/stage.yml +++ b/stage.yml @@ -24,8 +24,9 @@ services: user: django depends_on: - postgres - # - redis - command: /stage-gunicorn-mhackspace.sh + - directory + - redis + command: python /app/wsgi_bjoern.py env_file: .env volumes: - .:/app @@ -39,11 +40,12 @@ services: huey: build: context: . - dockerfile: ./compose/django/Dockerfile-dev + dockerfile: ./compose/django/Dockerfile user: django depends_on: - postgres - redis + - directory command: python manage.py run_huey env_file: .env volumes: diff --git a/wsgi_bjoern.py b/wsgi_bjoern.py index e16246a..e922128 100644 --- a/wsgi_bjoern.py +++ b/wsgi_bjoern.py @@ -52,13 +52,15 @@ application.debug = True # application = ReloadApplicationMiddleware(application) -socket_path = '/data/sockets/bjoern-mhackspace.sock' +socket_path = '/data/sockets/maidstone-hackspace.sock' if os.path.exists(socket_path): os.unlink(socket_path) + sock = socket.socket(socket.AF_UNIX) sock.bind(socket_path) sock.listen(1024) os.chmod(socket_path, 0o666) + try: bjoern.server_run(sock, application) except KeyboardInterrupt: