diff --git a/copier.yml b/copier.yml index cffdfe5..8a103c2 100644 --- a/copier.yml +++ b/copier.yml @@ -38,12 +38,6 @@ odoo_version: default: "{{branch_name}}" when: false -# Remove ? -use_secret: - type: bool - help: generate docker-compose dedicate file for secret - # only for bootstraping clea-ci|prod.secrets.docker-compose.yml - org_name: type: str default: Akretion diff --git a/src/.gitlab-ci.yml.jinja b/src/.gitlab-ci.yml.jinja index b6bea26..08ce5e6 100644 --- a/src/.gitlab-ci.yml.jinja +++ b/src/.gitlab-ci.yml.jinja @@ -64,9 +64,9 @@ before_script: - echo $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED - echo $CI_COMMIT_REF_SLUG - export MAIN_BRANCH=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}} - - export BUILD_NAME=${CI_PROJECT_NAME}--${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}} - - export CI_DOMAIN=${CI_DOMAIN} - - export DOMAIN=$BUILD_NAME.${CI_DOMAIN} + - export BUILD_NAME=${CI_PROJECT_NAME}_${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}} + - export PGDATABASE=$BUILD_NAME + - export DOMAIN=${CI_PROJECT_NAME}--${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}}.${CI_DOMAIN} prebuild: # warmup caches once per MR @@ -98,16 +98,12 @@ pregetdb: script: # no need to wait for a build # can be run from anywhere - # manage secrets - # run only if ci.secrets.docker-compose.yml exists - - test -f ci.secrets.docker-compose.yml && sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml - # if no secrets exists, create an empty one to make .env happy - - test -f secrets.docker-compose.yml || touch secrets.docker-compose.yml # TODO: factorise this: - cp .env-ci .env # TODO use copier or something to inialize this env variables - echo BUILD_NAME=${BUILD_NAME} >> .env + - echo PGDATABASE=${BUILD_NAME} >> .env - echo CI_PROJECT_NAME=${CI_PROJECT_NAME} >> .env - echo "DOMAIN=${DOMAIN}" >> .env # uid , gid used by docker in postgres to fix perms on .db @@ -139,6 +135,8 @@ fetch: - git remote add --mirror=fetch $REMOTE ~/$REMOTE || true - git fetch --all - git checkout ${NEW_HEAD} + - sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml + - ./bin/generate_env needs: ["prebuild"] build: @@ -147,19 +145,6 @@ build: # ak build then docker build - cd ~gitlab-runner/builds/${AK_WORKING_DIR} - - cp .env-ci .env - # TODO use copier or something to inialize this env variables - - echo BUILD_NAME=${BUILD_NAME} >> .env - - echo CI_PROJECT_NAME=${CI_PROJECT_NAME} >> .env - - echo "DOMAIN=${DOMAIN}" >> .env - # uid , gid used by docker in postgres to fix perms on .db - - echo "UID=`id -u`" >> .env - - echo "GID=`id -g`" >> .env - - # manage secrets - # run only if ci.secrets.docker-compose.yml exists - - test -f ci.secrets.docker-compose.yml && sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml - # ak build - cd odoo && time ak clone && time ak sparse && time ak build && cd .. @@ -310,55 +295,31 @@ lint: # can be run sooner needs: ["fetch"] - test: stage: test script: - CURRENT_PATH=`pwd` - cd ~gitlab-runner/builds/${AK_WORKING_DIR} - # get list of modules to test - # add back --select-exclude ${EXCLUDE_MODULES:-none} from a path - # qu'est ce que ça fait ça ? - #- docker compose run odoo initdb ${DB_NAME} --cache-prefix ${CI_PROJECT_NAME:0:7} - # TODO: réflechir si on laisse ça ici ou dans le conteneur - - # list of all local-src modules - - ADDONS_TEST=$(manifestoo --select-addons-dir odoo/local-src list --separator=,) - - ADDONS_TEST_DEP=$(manifestoo --select-addons-dir odoo/local-src list-depends --separator=,) - - echo $ADDONS_TEST - - echo $ADDONS_TEST_DEP - # start from a clean state - - docker compose --profile db run --rm bedrock dropdb --force --if-exists ${BUILD_NAME}_test - # --labels traefik.enable=false in order to not mess with the review - # init db with dependencies of our modules - # we force PG port in order to bypass pgbouncer - - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" -e "PGPORT=5432" -e "DB_PORT=5432" --rm odoo click-odoo-initdb -n ${BUILD_NAME}_test -m ${ADDONS_TEST_DEP} --cache-prefix ${CI_PROJECT_NAME:0:7} - # run the tests here - - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage run --include "/odoo/local-src/*" --branch /odoo/bin/odoo --stop-after-init -i ${ADDONS_TEST} --test-enable --workers=0 - # display report in MR - - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage report - # export for the artifacts - - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage xml --skip-empty - # extract coverage report - - docker compose cp odoo:/data/odoo/filestore/coverage.xml coverage.xml - # move coverage report to accessible location from gitlab-ci - - cp coverage.xml ${CURRENT_PATH}/coverage.xml + - docker compose --env-file .env-test kill + - dropdb --force --if-exists ${BUILD_NAME}_test + - docker compose --env-file .env-test run --rm odoo initdb ${BUILD_NAME}_test --cache-prefix ${CI_PROJECT_NAME:0:7} + - docker compose --env-file .env-test run --rm odoo runtests + - mv shared $CURRENT_PATH/ rules: # Run tests in MR if there is no Skiptest tag - if: $AK_IS_MR == "true" && $CI_MERGE_REQUEST_LABELS !~ /Skiptest/ + # Always run test in major branch + - if: $AK_IS_MAJOR_BRANCH == "true" && $AK_DO_OPENUPGRADE_MIGRATION != "true" needs: - # update_db do a docker compose stop, we don't want to be halted - # if this job start sooner - # optional true, let us the ability to run if update_db is cancelled - - job: "update_db" - optional: true - allow_failure: true - coverage: '/TOTAL(\W+\d+){5}\%/' + - job: "build" + coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: + when: always reports: + junit: shared/test_results/*.xml coverage_report: coverage_format: cobertura - path: coverage.xml + path: shared/coverage.xml # Run the container used for the review review: @@ -401,7 +362,7 @@ stop_review: review_preprod: stage: review_preprod script: - - echo "DB_NAME=$BUILD_NAME" >> .env + - echo "PGDATABASE=$BUILD_NAME" >> .env - echo "DOMAIN=${CI_PROJECT_NAME}--preprod-${CI_COMMIT_REF_SLUG}.${CI_DOMAIN}" >> .env - cd ~gitlab-runner/builds/${AK_WORKING_DIR} - docker compose up -d diff --git a/src/bin/generate_env b/src/bin/generate_env new file mode 100755 index 0000000..06745ac --- /dev/null +++ b/src/bin/generate_env @@ -0,0 +1,26 @@ +#!/bin/bash +set -euxo pipefail + +# generate env file (env, .env-test) +# Usage: +# ./generate_env + +echo "generate env file" +cp .env-ci .env +# uid , gid used by docker in postgres to fix perms on .db +cat << EOF >> .env +BUILD_NAME=${BUILD_NAME} +CI_PROJECT_NAME=${CI_PROJECT_NAME} +DOMAIN=${DOMAIN} +UID=`id -u` +GID=`id -g` +EOF + +cp .env .env-test +echo "PGDATABASE=${BUILD_NAME}" >> .env + +cat << EOF >> .env +ENABLE_TRAEFIK=false +COMPOSE_PROJECT_NAME=${BUILD_NAME}_test +PGDATABASE=${BUILD_NAME}_test +EOF diff --git a/src/ci.docker-compose.yml.jinja b/src/ci.docker-compose.yml.jinja index 4db49ac..f185412 100644 --- a/src/ci.docker-compose.yml.jinja +++ b/src/ci.docker-compose.yml.jinja @@ -31,19 +31,16 @@ services: # but take care to use a different DSN than the production PGSSLMODE: require - DB_NAME: ${BUILD_NAME} - PGDATABASE: ${BUILD_NAME} - DB_USER: ${CI_PROJECT_NAME} - PGUSER: ${CI_PROJECT_NAME} - DB_HOST: - PGHOST: + PGDATABASE: $PGDATABASE + PGHOST: $PGHOST + PGPASSWORD: $PGPASSWORD + PGUSER: $PGUSER {% if branch_name != "18.0" %} # use pgbouncer - DB_PORT: 6432 PGPORT: 6432 {% endif %} labels: - - "traefik.enable=true" + - "traefik.enable=${ENABLE_TRAEFIK:-true}" - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.rule=Host(`${DOMAIN}`)" - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.tls=false" - "traefik.http.routers.${COMPOSE_PROJECT_NAME}-long.rule=Host(`${DOMAIN}`) && PathPrefix(`/longpolling/`)" @@ -65,7 +62,7 @@ services: target: /data/odoo/filestore - type: bind source: ./shared - target: /data/odoo/shared + target: /odoo/shared bind: # create shared dir if needed create_host_path: true @@ -73,16 +70,11 @@ services: image: ghcr.io/akretion/odoo-docker:{{ odoo_version }}-light-latest profiles: [db] environment: - DB_NAME: ${BUILD_NAME} - PGDATABASE: ${BUILD_NAME} - DB_USER: ${CI_PROJECT_NAME} - PGUSER: ${CI_PROJECT_NAME} - DB_HOST: - PGHOST: - {% if not use_secret %} - PGPASSWORD: ${PGPASSWORD} - DB_PASSWORD: ${PGPASSWORD} - {% endif %} + PGSSLMODE: require + PGDATABASE: $PGDATABASE + PGHOST: $PGHOST + PGPASSWORD: $PGPASSWORD + PGUSER: $PGUSER # we give AK_TEMPLATE_DB here for get_spare AK_TEMPLATE_DB: ${AK_TEMPLATE_DB} volumes: diff --git a/src/{% if use_secret %}clear-ci.secrets.docker-compose.yml{% endif %}.jinja b/src/clear-ci.secrets.docker-compose.yml.jinja similarity index 80% rename from src/{% if use_secret %}clear-ci.secrets.docker-compose.yml{% endif %}.jinja rename to src/clear-ci.secrets.docker-compose.yml.jinja index 65392a6..c3ea786 100644 --- a/src/{% if use_secret %}clear-ci.secrets.docker-compose.yml{% endif %}.jinja +++ b/src/clear-ci.secrets.docker-compose.yml.jinja @@ -5,12 +5,6 @@ services: odoo: environment: - DB_PASSWORD: - PGPASSWORD: ENCRYPTION_KEY_CI: AUTH_OAUTH_PROVIDER_CLIENT_ID: {{ project_name }} AUTH_OAUTH_PROVIDER_CLIENT_SECRET: - bedrock: - environment: - DB_PASSWORD: - PGPASSWORD: diff --git a/src/{% if use_secret %}clear-prod.secrets.docker-compose.yml{% endif %}.jinja b/src/clear-prod.secrets.docker-compose.yml.jinja similarity index 90% rename from src/{% if use_secret %}clear-prod.secrets.docker-compose.yml{% endif %}.jinja rename to src/clear-prod.secrets.docker-compose.yml.jinja index 77a3b51..e2611fb 100644 --- a/src/{% if use_secret %}clear-prod.secrets.docker-compose.yml{% endif %}.jinja +++ b/src/clear-prod.secrets.docker-compose.yml.jinja @@ -4,8 +4,6 @@ services: odoo: environment: - DB_PASSWORD: - PGPASSWORD: ENCRYPTION_KEY_PROD: ENCRYPTION_KEY_CI: AUTH_OAUTH_PROVIDER_CLIENT_ID: {{ project_name }} diff --git a/src/dev.docker-compose.yml.jinja b/src/dev.docker-compose.yml.jinja index 80dc668..0c4d5d1 100644 --- a/src/dev.docker-compose.yml.jinja +++ b/src/dev.docker-compose.yml.jinja @@ -37,11 +37,6 @@ services: # you may want to add queue_job here SERVER_WIDE_MODULES: web - # DB_ and PG specified because entrypoint is not always ran if docker exec - DB_USER: odoo - DB_HOST: db - DB_PASSWORD: odoo - DB_NAME: db PGDATABASE: db PGUSER: odoo PGHOST: db diff --git a/src/odoo/Dockerfile.jinja b/src/odoo/Dockerfile.jinja index 8130aaa..bacda56 100644 --- a/src/odoo/Dockerfile.jinja +++ b/src/odoo/Dockerfile.jinja @@ -22,6 +22,8 @@ COPY ./src/odoo/tools /odoo/src/odoo/tools COPY ./src/odoo/upgrade /odoo/src/odoo/upgrade RUN /odoo/bin/pip install -e /odoo/src +COPY bin/* /usr/local/bin/ + FROM base as thisproject COPY ./requirements.txt /odoo/requirements.txt RUN /odoo/bin/pip install --no-cache-dir \ diff --git a/src/odoo/bin/initdb b/src/odoo/bin/initdb new file mode 100755 index 0000000..de59ced --- /dev/null +++ b/src/odoo/bin/initdb @@ -0,0 +1,16 @@ +#!/bin/bash +set -euxo pipefail + +# Usage: initdb dbtest + +echo "Create database $1" + +EXTRA_ARGS="${@:2}" +ADDONS_INIT=$(manifestoo --select-exclude ${EXCLUDE_MODULES:-none} --select-addons-dir /odoo/local-src list-depends --separator=,) + +# Some module like product_code_unique can break the installation if their are installed too early, so you can use the variable "EXCLUDE_DEPENDENCIES" to exclude them + +export ADDONS_INIT +ADDONS_INIT=l10n_generic_coa,$(python3 -c "import os; print(','.join([x for x in os.environ.get('ADDONS_INIT').split(',') if x not in os.environ.get('EXCLUDE_DEPENDENCIES', '').split(',')]))") + +click-odoo-initdb -n $1 -m ${ADDONS_INIT} ${EXTRA_ARGS} diff --git a/src/odoo/bin/runtests b/src/odoo/bin/runtests new file mode 100755 index 0000000..25ba7b5 --- /dev/null +++ b/src/odoo/bin/runtests @@ -0,0 +1,19 @@ +#!/bin/bash +set -euxo pipefail + +# Usage: runtest + +# Run tests. Use unbuffer to get a colored output. + +ADDONS_TEST=$(manifestoo --select-exclude ${EXCLUDE_MODULES:-none} --select-addons-dir /odoo/local-src list --separator=,) +# Move to shared folder so coverage file will be shared with the host +# and so accessible by the gitlab runner +cd /odoo/shared +unbuffer coverage run --include "/odoo/local-src/*" --branch $(which odoo) --stop-after-init -i ${ADDONS_TEST} --test-enable --workers=0 +ls -alh +coverage report -i --precision=2 --skip-empty +coverage xml -i -o coverage.xml + +sed -i "s==/home/gitlab-runner/builds=g" /odoo/shared/coverage.xml +sed -i "s=/odoo/local-src=${CI_PROJECT_DIR}/odoo/local-src=g" /odoo/shared/coverage.xml +sed -i "s=/home/gitlab-runner/builds/=builds/=g" /odoo/shared/coverage.xml diff --git a/src/odoo/local-src/{{ project_name }}/__init__.py b/src/odoo/local-src/custom_all/__init__.py similarity index 100% rename from src/odoo/local-src/{{ project_name }}/__init__.py rename to src/odoo/local-src/custom_all/__init__.py diff --git a/src/odoo/local-src/{{ project_name }}/__manifest__.py.jinja b/src/odoo/local-src/custom_all/__manifest__.py.jinja similarity index 95% rename from src/odoo/local-src/{{ project_name }}/__manifest__.py.jinja rename to src/odoo/local-src/custom_all/__manifest__.py.jinja index a5e576c..ac47e94 100644 --- a/src/odoo/local-src/{{ project_name }}/__manifest__.py.jinja +++ b/src/odoo/local-src/custom_all/__manifest__.py.jinja @@ -16,6 +16,7 @@ "auth_oidc_akretion_data", "auth_oidc_environment", "database_age_cron", + "odoo_test_xmlrunner", "server_environment_ir_config_parameter", "web_environment_ribbon", ], diff --git a/src/odoo/requirements.txt b/src/odoo/requirements.txt index 5ed0b2f..142fad4 100644 --- a/src/odoo/requirements.txt +++ b/src/odoo/requirements.txt @@ -3,6 +3,7 @@ pyjwt # auth_oidc python-jose[cryptography] # auth_oidc #sentry 1.9.0 fixé dans le manifest du module sentry pour le moment sentry-sdk<=1.9.0 +unittest-xml-reporting # odoo_test_xmlrunner #apispec #shopinvader #boto3==1.20.32 # shopinvader diff --git a/src/odoo/spec.yaml.jinja b/src/odoo/spec.yaml.jinja index 1aebaa2..1621717 100644 --- a/src/odoo/spec.yaml.jinja +++ b/src/odoo/spec.yaml.jinja @@ -25,6 +25,7 @@ server-tools: modules: - sentry - bus_alt_connection + - odoo_test_xmlrunner src: https://github.com/oca/server-tools {{ odoo_version }} web: diff --git a/src/prod.docker-compose.yml.jinja b/src/prod.docker-compose.yml.jinja index ad98cb9..1b6a0e5 100644 --- a/src/prod.docker-compose.yml.jinja +++ b/src/prod.docker-compose.yml.jinja @@ -39,17 +39,13 @@ services: SERVER_WIDE_MODULES: web,sentry,bus_alt_connection,queue_job {% endif %} - # ask infra team for the following PGSSLMODE: require - DB_NAME: - PGDATABASE: - DB_HOST: - PGHOST: - PGUSER: - DB_USER: + PGDATABASE: $PGDATABASE + PGHOST: $PGHOST + PGPASSWORD: $PGPASSWORD + PGUSER: $PGUSER {% if branch_name != "18.0" %} # use pgbouncer - DB_PORT: 6432 PGPORT: 6432 {% endif %} diff --git a/src/{% if use_secret %}secrets.docker-compose.yml{% endif %}.jinja b/src/{% if use_secret %}secrets.docker-compose.yml{% endif %}.jinja deleted file mode 100644 index ef0b20c..0000000 --- a/src/{% if use_secret %}secrets.docker-compose.yml{% endif %}.jinja +++ /dev/null @@ -1,21 +0,0 @@ -# never commit this file -# only for dev -services: - odoo: - environment: - # usually you want a lax addons path in dev - # and a strict (only authorized modules) in prod - - ADDONS_PATH=/odoo/links,/odoo/local-src,/odoo/src/odoo/addons,/odoo/src/addons - - DB_HOST=db - - DB_NAME=db - - DB_PASSWORD=odoo - - DB_USER=odoo - - PGDATABASE=db - - PGHOST=db - - PGPASSWORD=odoo - - PGUSER=odoo - db: - environment: - - POSTGRES_DB=db - - POSTGRES_PASSWORD=odoo - - POSTGRES_USER=odoo