diff --git a/.github/workflows/add_column_BQ_ds_cd.yml b/.github/workflows/add_column_BQ_ds_cd.yml deleted file mode 100644 index 6b35ded9..00000000 --- a/.github/workflows/add_column_BQ_ds_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'add_column_BQ_ds/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./add_column_BQ_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_BQ_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_BQ_ds_ci.yml b/.github/workflows/add_column_BQ_ds_ci.yml deleted file mode 100644 index b1db2bff..00000000 --- a/.github/workflows/add_column_BQ_ds_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'add_column_BQ_ds/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./add_column_BQ_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_BQ_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_kafka_data_source_cd.yml b/.github/workflows/add_column_kafka_data_source_cd.yml deleted file mode 100644 index 6bcf79bf..00000000 --- a/.github/workflows/add_column_kafka_data_source_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'add_column_kafka_data_source/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./add_column_kafka_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_KAFKA_DATA_SOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_kafka_data_source_ci.yml b/.github/workflows/add_column_kafka_data_source_ci.yml deleted file mode 100644 index b1c68b3f..00000000 --- a/.github/workflows/add_column_kafka_data_source_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'add_column_kafka_data_source/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./add_column_kafka_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_KAFKA_DATA_SOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_snowflake_ds_cd.yml b/.github/workflows/add_column_snowflake_ds_cd.yml deleted file mode 100644 index 893b814f..00000000 --- a/.github/workflows/add_column_snowflake_ds_cd.yml +++ /dev/null @@ -1,22 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CD Workflow - -on: - workflow_dispatch: - push: - paths: - - 'add_column_snowflake_ds/**' - branches: - - main -jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./add_column_snowflake_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_snowflake_ds_ci.yml b/.github/workflows/add_column_snowflake_ds_ci.yml deleted file mode 100644 index a156cfc5..00000000 --- a/.github/workflows/add_column_snowflake_ds_ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CI Workflow - -on: - workflow_dispatch: - pull_request: - paths: - - 'add_column_snowflake_ds/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - -concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - -jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./add_column_snowflake_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_to_materialized_view_cd.yml b/.github/workflows/add_column_to_materialized_view_cd.yml deleted file mode 100644 index 1166b9d4..00000000 --- a/.github/workflows/add_column_to_materialized_view_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'add_new_column_to_a_materialized_view/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./add_new_column_to_a_materialized_view - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_TO_MATERIALIZED_VIEW }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_to_materialized_view_ci.yml b/.github/workflows/add_column_to_materialized_view_ci.yml deleted file mode 100644 index 18afa92f..00000000 --- a/.github/workflows/add_column_to_materialized_view_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'add_new_column_to_a_materialized_view/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./add_new_column_to_a_materialized_view - use_last_partition: false - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_TO_MATERIALIZED_VIEW }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_to_s3_ds_cd.yml b/.github/workflows/add_column_to_s3_ds_cd.yml deleted file mode 100644 index dae98052..00000000 --- a/.github/workflows/add_column_to_s3_ds_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'add_column_to_s3_ds/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./add_column_to_s3_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_TO_S3_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_column_to_s3_ds_ci.yml b/.github/workflows/add_column_to_s3_ds_ci.yml deleted file mode 100644 index c9c8bd72..00000000 --- a/.github/workflows/add_column_to_s3_ds_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'add_column_to_s3_ds/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./add_column_to_s3_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_TO_S3_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_new_column_with_values_cd.yml b/.github/workflows/add_new_column_with_values_cd.yml deleted file mode 100644 index 1e5c878d..00000000 --- a/.github/workflows/add_new_column_with_values_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'add_new_column_with_values/**' - jobs: - cd: # deploy changes to workspace 'add_new_column_with_values' - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: add_new_column_with_values - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_NEW_COLUMN_WITH_VALUES }} # set the Workspace admin token in GitHub secrets - tb_host: https://ui.tinybird.co diff --git a/.github/workflows/add_new_column_with_values_ci.yml b/.github/workflows/add_new_column_with_values_ci.yml deleted file mode 100644 index f9a014f6..00000000 --- a/.github/workflows/add_new_column_with_values_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'add_new_column_with_values/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using branches from workspace 'add_new_column_with_values' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: add_new_column_with_values - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_NEW_COLUMN_WITH_VALUES }} # set the Workspace admin token in GitHub secrets - tb_host: https://ui.tinybird.co diff --git a/.github/workflows/add_nullable_column_to_landing_data_source_cd.yml b/.github/workflows/add_nullable_column_to_landing_data_source_cd.yml deleted file mode 100644 index ea4c7a7a..00000000 --- a/.github/workflows/add_nullable_column_to_landing_data_source_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'add_nullable_column_to_landing_data_source/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: add_nullable_column_to_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_NULLABLE_COLUMN_LANDING_DS }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/add_nullable_column_to_landing_data_source_ci.yml b/.github/workflows/add_nullable_column_to_landing_data_source_ci.yml deleted file mode 100644 index 20b270f0..00000000 --- a/.github/workflows/add_nullable_column_to_landing_data_source_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'add_nullable_column_to_landing_data_source/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: add_nullable_column_to_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_NULLABLE_COLUMN_LANDING_DS }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/alter_column_to_nullable_cd.yml b/.github/workflows/alter_column_to_nullable_cd.yml deleted file mode 100644 index ffa09d49..00000000 --- a/.github/workflows/alter_column_to_nullable_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'alter_column_to_nullable/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.1 - with: - data_project_dir: ./alter_column_to_nullable - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ALTER_NULLABLE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/alter_column_to_nullable_ci.yml b/.github/workflows/alter_column_to_nullable_ci.yml deleted file mode 100644 index 6dd5e8de..00000000 --- a/.github/workflows/alter_column_to_nullable_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'alter_column_to_nullable/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.1 - with: - data_project_dir: ./alter_column_to_nullable - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ALTER_NULLABLE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/alter_default_cd.yml b/.github/workflows/alter_default_cd.yml deleted file mode 100644 index 68770a9d..00000000 --- a/.github/workflows/alter_default_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'alter_default_value_in_column/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.1 - with: - data_project_dir: ./alter_default_value_in_column - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ALTER_DEFAULT }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/alter_default_ci.yml b/.github/workflows/alter_default_ci.yml deleted file mode 100644 index 22d60d46..00000000 --- a/.github/workflows/alter_default_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'alter_default_value_in_column/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.1 - with: - data_project_dir: ./alter_default_value_in_column - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ALTER_DEFAULT }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_column_type_landing_and_recover_quarantine_data_cd.yml b/.github/workflows/change_column_type_landing_and_recover_quarantine_data_cd.yml deleted file mode 100644 index ab60101c..00000000 --- a/.github/workflows/change_column_type_landing_and_recover_quarantine_data_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_column_type_landing_and_recover_quarantine_data/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_column_type_landing_and_recover_quarantine_data - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_COLUMN_TYPE_LANDING_AND_RECOVER_QUARANTINE_DATA}} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_column_type_landing_and_recover_quarantine_data_ci.yml b/.github/workflows/change_column_type_landing_and_recover_quarantine_data_ci.yml deleted file mode 100644 index f4df7983..00000000 --- a/.github/workflows/change_column_type_landing_and_recover_quarantine_data_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_column_type_landing_and_recover_quarantine_data/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_column_type_landing_and_recover_quarantine_data - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_COLUMN_TYPE_LANDING_AND_RECOVER_QUARANTINE_DATA }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_column_type_materialized_view_cd.yml b/.github/workflows/change_column_type_materialized_view_cd.yml deleted file mode 100644 index 8f1acb11..00000000 --- a/.github/workflows/change_column_type_materialized_view_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'change_column_type_materialized_view/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: change_column_type_materialized_view - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CHANGE_COLUMN_TYPE_MATERIALIZED_VIEW }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_column_type_materialized_view_ci.yml b/.github/workflows/change_column_type_materialized_view_ci.yml deleted file mode 100644 index 8751ba9f..00000000 --- a/.github/workflows/change_column_type_materialized_view_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'change_column_type_materialized_view/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: change_column_type_materialized_view - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CHANGE_COLUMN_TYPE_MATERIALIZED_VIEW }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_copy_pipe_time_granularity_cd.yml b/.github/workflows/change_copy_pipe_time_granularity_cd.yml deleted file mode 100644 index 9e40f42f..00000000 --- a/.github/workflows/change_copy_pipe_time_granularity_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_copy_pipe_time_granularity/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_copy_pipe_time_granularity - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_COPY_PIPE_TIME_GRANULARITY }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_copy_pipe_time_granularity_ci.yml b/.github/workflows/change_copy_pipe_time_granularity_ci.yml deleted file mode 100644 index f4187482..00000000 --- a/.github/workflows/change_copy_pipe_time_granularity_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_copy_pipe_time_granularity/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_copy_pipe_time_granularity - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_COPY_PIPE_TIME_GRANULARITY }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_data_source_ttl_cd.yml b/.github/workflows/change_data_source_ttl_cd.yml deleted file mode 100644 index 2a82f07f..00000000 --- a/.github/workflows/change_data_source_ttl_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_data_source_ttl/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_data_source_ttl - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_DATA_SOURCE_TTL }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_data_source_ttl_ci.yml b/.github/workflows/change_data_source_ttl_ci.yml deleted file mode 100644 index 69e63506..00000000 --- a/.github/workflows/change_data_source_ttl_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_data_source_ttl/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_data_source_ttl - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_DATA_SOURCE_TTL }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sk_to_kafka_data_source_cd.yml b/.github/workflows/change_sk_to_kafka_data_source_cd.yml deleted file mode 100644 index 73820083..00000000 --- a/.github/workflows/change_sk_to_kafka_data_source_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'change_sorting_key_kafka_data_source/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: change_sorting_key_kafka_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_KAFKA_DATA_SOURCE }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sk_to_kafka_data_source_ci.yml b/.github/workflows/change_sk_to_kafka_data_source_ci.yml deleted file mode 100644 index 2539c87e..00000000 --- a/.github/workflows/change_sk_to_kafka_data_source_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'change_sorting_key_kafka_data_source/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: change_sorting_key_kafka_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_KAFKA_DATA_SOURCE }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sorting_key_landing_data_source_cd.yml b/.github/workflows/change_sorting_key_landing_data_source_cd.yml deleted file mode 100644 index fbc931e1..00000000 --- a/.github/workflows/change_sorting_key_landing_data_source_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_sorting_key_landing_data_source/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_LANDING_DATA_SOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sorting_key_landing_data_source_ci.yml b/.github/workflows/change_sorting_key_landing_data_source_ci.yml deleted file mode 100644 index e211ef71..00000000 --- a/.github/workflows/change_sorting_key_landing_data_source_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_sorting_key_landing_data_source/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_LANDING_DATA_SOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sorting_key_to_s3_data_source_cd.yml b/.github/workflows/change_sorting_key_to_s3_data_source_cd.yml deleted file mode 100644 index b6330878..00000000 --- a/.github/workflows/change_sorting_key_to_s3_data_source_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_sorting_key_to_s3_data_source/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_to_s3_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_S3_DATA_SOURCE }} - tb_host: https://ui.wadus2.gcp.tinybird.co diff --git a/.github/workflows/change_sorting_key_to_s3_data_source_ci.yml b/.github/workflows/change_sorting_key_to_s3_data_source_ci.yml deleted file mode 100644 index 7a8e07bc..00000000 --- a/.github/workflows/change_sorting_key_to_s3_data_source_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_sorting_key_to_s3_data_source/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_to_s3_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_S3_DATA_SOURCE }} - tb_host: https://ui.wadus2.gcp.tinybird.co diff --git a/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_cd.yml b/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_cd.yml deleted file mode 100644 index a502f8ed..00000000 --- a/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'change_sorting_key_to_s3_data_source_with_reingestion/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_to_s3_data_source_with_reingestion - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_S3_DATA_SOURCE_WITH_REINGESTION }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_ci.yml b/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_ci.yml deleted file mode 100644 index 52e2ce07..00000000 --- a/.github/workflows/change_sorting_key_to_s3_data_source_with_reingestion_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'change_sorting_key_to_s3_data_source_with_reingestion/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./change_sorting_key_to_s3_data_source_with_reingestion - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CHANGE_SORTING_KEY_TO_S3_DATA_SOURCE_WITH_REINGESTION }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/create_a_materialized_view_batch_ingest_cd.yml b/.github/workflows/create_a_materialized_view_batch_ingest_cd.yml deleted file mode 100644 index fad668e8..00000000 --- a/.github/workflows/create_a_materialized_view_batch_ingest_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'create_a_materialized_view_batch_ingest/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./create_a_materialized_view_batch_ingest - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CREATE_A_MATERIALIZED_VIEW_BATCH_INGEST }} - tb_host: https://api.us-east.tinybird.co \ No newline at end of file diff --git a/.github/workflows/create_a_materialized_view_batch_ingest_ci.yml b/.github/workflows/create_a_materialized_view_batch_ingest_ci.yml deleted file mode 100644 index 1302aa80..00000000 --- a/.github/workflows/create_a_materialized_view_batch_ingest_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'create_a_materialized_view_batch_ingest/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./create_a_materialized_view_batch_ingest - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CREATE_A_MATERIALIZED_VIEW_BATCH_INGEST }} - tb_host: https://api.us-east.tinybird.co diff --git a/.github/workflows/create_pipe_sink_cd.yml b/.github/workflows/create_pipe_sink_cd.yml deleted file mode 100644 index 733ff791..00000000 --- a/.github/workflows/create_pipe_sink_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'create_pipe_sink/**' - jobs: - cd: # deploy changes to workspace 'create_pipe_sink' - uses: tinybirdco/ci/.github/workflows/cd.yml@v3.1.0 - with: - data_project_dir: create_pipe_sink - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CREATE_PIPE_SINK_TOKEN }} # set the Workspace admin token in GitHub secrets - tb_host: https://ui.us-east.tinybird.co diff --git a/.github/workflows/create_pipe_sink_ci.yml b/.github/workflows/create_pipe_sink_ci.yml deleted file mode 100644 index e6822353..00000000 --- a/.github/workflows/create_pipe_sink_ci.yml +++ /dev/null @@ -1,29 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'create_pipe_sink/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using branches from workspace 'create_pipe_sink' - uses: tinybirdco/ci/.github/workflows/ci.yml@v3.1.0 - with: - data_project_dir: create_pipe_sink - tb_format: false - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_CREATE_PIPE_SINK_TOKEN }} # set the Workspace admin token in GitHub secrets - tb_host: https://ui.us-east.tinybird.co diff --git a/.github/workflows/create_s3_ds_cd.yml b/.github/workflows/create_s3_ds_cd.yml deleted file mode 100644 index eb49eaea..00000000 --- a/.github/workflows/create_s3_ds_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'create_s3_datasource/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.1 - with: - data_project_dir: ./create_s3_datasource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/create_s3_ds_ci.yml b/.github/workflows/create_s3_ds_ci.yml deleted file mode 100644 index 623a15de..00000000 --- a/.github/workflows/create_s3_ds_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'create_s3_datasource/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.1 - with: - data_project_dir: ./create_s3_datasource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_CREATE_S3_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_column_materialized_view_cd.yml b/.github/workflows/delete_column_materialized_view_cd.yml deleted file mode 100644 index 5ac743a5..00000000 --- a/.github/workflows/delete_column_materialized_view_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'delete_column_materialized_view/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./delete_column_materialized_view - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_COLUMN_MV }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_column_materialized_view_ci.yml b/.github/workflows/delete_column_materialized_view_ci.yml deleted file mode 100644 index 7568d098..00000000 --- a/.github/workflows/delete_column_materialized_view_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'delete_column_materialized_view/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./delete_column_materialized_view - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_COLUMN_MV }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_column_snowflake_ds_cd.yml b/.github/workflows/delete_column_snowflake_ds_cd.yml deleted file mode 100644 index 2ad50790..00000000 --- a/.github/workflows/delete_column_snowflake_ds_cd.yml +++ /dev/null @@ -1,22 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CD Workflow - -on: - workflow_dispatch: - push: - paths: - - 'delete_column_snowflake_ds/**' - branches: - - main -jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./delete_column_snowflake_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_column_snowflake_ds_ci.yml b/.github/workflows/delete_column_snowflake_ds_ci.yml deleted file mode 100644 index 815f397d..00000000 --- a/.github/workflows/delete_column_snowflake_ds_ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CI Workflow - -on: - workflow_dispatch: - pull_request: - paths: - - 'delete_column_snowflake_ds/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - -concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - -jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./delete_column_snowflake_ds - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_simple_resource_cd.yml b/.github/workflows/delete_simple_resource_cd.yml deleted file mode 100644 index 7359e1e8..00000000 --- a/.github/workflows/delete_simple_resource_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'delete_simple_resource/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./delete_simple_resource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/delete_simple_resource_ci.yml b/.github/workflows/delete_simple_resource_ci.yml deleted file mode 100644 index b7de4ea2..00000000 --- a/.github/workflows/delete_simple_resource_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'delete_simple_resource/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./delete_simple_resource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/drop_column_cd.yml b/.github/workflows/drop_column_cd.yml deleted file mode 100644 index 689240a2..00000000 --- a/.github/workflows/drop_column_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'drop_column/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.1 - with: - data_project_dir: ./drop_column - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DROP_COLUMN }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/drop_column_ci.yml b/.github/workflows/drop_column_ci.yml deleted file mode 100644 index 51d53a69..00000000 --- a/.github/workflows/drop_column_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'drop_column/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.1 - with: - data_project_dir: ./drop_column - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DROP_COLUMN }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/how_to_rollback_cd.yml b/.github/workflows/how_to_rollback_cd.yml deleted file mode 100644 index 90fa7662..00000000 --- a/.github/workflows/how_to_rollback_cd.yml +++ /dev/null @@ -1,23 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - paths: - - 'how_to_rollback/**' - branches: - - main - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./how_to_rollback - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_HOW_TO_ROLLBACK }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/how_to_rollback_ci.yml b/.github/workflows/how_to_rollback_ci.yml deleted file mode 100644 index d9578a4c..00000000 --- a/.github/workflows/how_to_rollback_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - paths: - - 'how_to_rollback/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./how_to_rollback - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_HOW_TO_ROLLBACK }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterate_dynamodb_cd.yml b/.github/workflows/iterate_dynamodb_cd.yml deleted file mode 100644 index f0524657..00000000 --- a/.github/workflows/iterate_dynamodb_cd.yml +++ /dev/null @@ -1,22 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CD Workflow - -on: - workflow_dispatch: - push: - paths: - - 'iterate_dynamodb/**' - branches: - - main -jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 - with: - data_project_dir: ./iterate_dynamodb - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ITERATIONS }} - tb_host: https://api.us-east.aws.tinybird.co diff --git a/.github/workflows/iterate_dynamodb_ci.yml b/.github/workflows/iterate_dynamodb_ci.yml deleted file mode 100644 index ee8650c0..00000000 --- a/.github/workflows/iterate_dynamodb_ci.yml +++ /dev/null @@ -1,27 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CI Workflow - -on: - workflow_dispatch: - pull_request: - paths: - - 'iterate_dynamodb/**' - branches: - - main - - '*-dynamo-*' - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - -concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - -jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 - with: - data_project_dir: ./iterate_dynamodb - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ITERATIONS }} - tb_host: https://api.us-east.aws.tinybird.co diff --git a/.github/workflows/iterate_snowflake_cd.yml b/.github/workflows/iterate_snowflake_cd.yml deleted file mode 100644 index ebeb19c2..00000000 --- a/.github/workflows/iterate_snowflake_cd.yml +++ /dev/null @@ -1,22 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CD Workflow - -on: - workflow_dispatch: - push: - paths: - - 'iterate_snowflake/**' - branches: - - main -jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.1.0 - with: - data_project_dir: ./iterate_snowflake - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterate_snowflake_ci.yml b/.github/workflows/iterate_snowflake_ci.yml deleted file mode 100644 index f0e6496f..00000000 --- a/.github/workflows/iterate_snowflake_ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -################################################## -### Visit https://github.com/tinybirdco/ci ### -### for more details or custom CI/CD ### -################################################## - -name: Tinybird - CI Workflow - -on: - workflow_dispatch: - pull_request: - paths: - - 'iterate_snowflake/**' - branches: - - main - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - -concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - -jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.1.0 - with: - data_project_dir: ./iterate_snowflake - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_ADD_COLUMN_SNOWFLAKE_DS }} - tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_cd.yml b/.github/workflows/iterating_api_endpoint_cd.yml deleted file mode 100644 index bc60e286..00000000 --- a/.github/workflows/iterating_api_endpoint_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'iterating_api_endpoint/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: iterating_api_endpoint - secrets: - tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/iterating_api_endpoint_ci.yml b/.github/workflows/iterating_api_endpoint_ci.yml deleted file mode 100644 index d78480be..00000000 --- a/.github/workflows/iterating_api_endpoint_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'iterating_api_endpoint/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using environments from workspace 'iterating_api_endpoint' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: iterating_api_endpoint - secrets: - tb_admin_token: ${{ secrets.ITERATING_API_ENDPOINT_TOKEN }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/recover_data_from_quarantine_cd.yml b/.github/workflows/recover_data_from_quarantine_cd.yml deleted file mode 100644 index ee2d7527..00000000 --- a/.github/workflows/recover_data_from_quarantine_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'recover_data_from_quarantine/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: ./recover_data_from_quarantine/* - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_QUARANTINE_V3 }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/recover_data_from_quarantine_ci.yml b/.github/workflows/recover_data_from_quarantine_ci.yml deleted file mode 100644 index e6b3534f..00000000 --- a/.github/workflows/recover_data_from_quarantine_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'recover_data_from_quarantine/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using environments from workspace 'quarentine' - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: ./recover_data_from_quarantine - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_QUARANTINE_V3 }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/remove_column_landing_data_source_cd.yml b/.github/workflows/remove_column_landing_data_source_cd.yml deleted file mode 100644 index 7f01674b..00000000 --- a/.github/workflows/remove_column_landing_data_source_cd.yml +++ /dev/null @@ -1,24 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'remove_column_landing_data_source/**' - jobs: - cd: - uses: tinybirdco/ci/.github/workflows/cd.yml@v4.0.0 - with: - data_project_dir: remove_column_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_REMOVE_COLUMN_LANDING_DATA_SOURCE }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/remove_column_landing_data_source_ci.yml b/.github/workflows/remove_column_landing_data_source_ci.yml deleted file mode 100644 index 52074a77..00000000 --- a/.github/workflows/remove_column_landing_data_source_ci.yml +++ /dev/null @@ -1,28 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'remove_column_landing_data_source/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: - uses: tinybirdco/ci/.github/workflows/ci.yml@v4.0.0 - with: - data_project_dir: remove_column_landing_data_source - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_REMOVE_COLUMN_LANDING_DATA_SOURCE }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/tinybird-ci.yml b/.github/workflows/tinybird-ci.yml new file mode 100644 index 00000000..c8879eff --- /dev/null +++ b/.github/workflows/tinybird-ci.yml @@ -0,0 +1,210 @@ +name: Tinybird - CI Workflow + +on: + workflow_dispatch: + pull_request: + branches: + - main + - master + - local_env_refactor + types: [opened, reopened, labeled, unlabeled, synchronize, closed] + +concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} + +jobs: + add_new_column_to_a_materialized_view: + runs-on: ubuntu-latest + defaults: + run: + working-directory: './add_new_column_to_a_materialized_view' + services: + tinybird: + image: tinybirdco/tinybird-local:beta + ports: + - 80:80 + steps: + - uses: actions/checkout@v3 + - name: Set local environment credentials (required for local deployments) + run: | + echo "TB_HOST=http://localhost:80" >> $GITHUB_ENV + echo "TB_TOKEN=$(curl -s http://localhost:80/tokens | jq -r .admin_token)" >> $GITHUB_ENV + - name: Install Tinybird CLI + run: curl -LsSf https://api.tinybird.co/static/install.sh | sh + - name: Build project + run: tb build + - name: Test project + run: tb test run + - name: deploy project + run: tb deployment create --auto --wait + - name: push some data + run: tb datasource append analytics_events fixtures/analytics_events.ndjson + - name: check data in the MV + run: tb sql "select count() from analytics_sessions_mv1" + - name: apply patch + run: cp -r .diff/* . && git diff + - name: deploy project again + run: tb deployment create --auto --wait + - name: check data in the MV after deploy + run: tb sql "select count() from analytics_sessions_mv1" + + + change_sorting_key_landing_data_source_2: + runs-on: ubuntu-latest + defaults: + run: + working-directory: './change_sorting_key_landing_data_source_2' + services: + tinybird: + image: tinybirdco/tinybird-local:beta + ports: + - 80:80 + steps: + - uses: actions/checkout@v3 + - name: Set local environment credentials (required for local deployments) + run: | + echo "TB_HOST=http://localhost:80" >> $GITHUB_ENV + echo "TB_TOKEN=$(curl -s http://localhost:80/tokens | jq -r .admin_token)" >> $GITHUB_ENV + - name: Install Tinybird CLI + run: curl -LsSf https://api.tinybird.co/static/install.sh | sh + - name: Build project + run: tb build + - name: Test project + run: tb test run + - name: deploy project + run: tb deployment create --auto --wait + - name: push some data + run: tb datasource append analytics_events fixtures/analytics_events.ndjson + - name: check data in the MV + run: tb sql "select count() from analytics_events" + - name: apply patch + run: cp -r .diff/* . && git diff + - name: deploy project again + run: tb deployment create --auto --wait + - name: check data in the MV after deploy + run: tb sql "select count() from analytics_events" + + + delete_simple_resource: + runs-on: ubuntu-latest + defaults: + run: + working-directory: './delete_simple_resource' + services: + tinybird: + image: tinybirdco/tinybird-local:beta + ports: + - 80:80 + steps: + - uses: actions/checkout@v3 + - name: Set local environment credentials (required for local deployments) + run: | + echo "TB_HOST=http://localhost:80" >> $GITHUB_ENV + echo "TB_TOKEN=$(curl -s http://localhost:80/tokens | jq -r .admin_token)" >> $GITHUB_ENV + - name: Install Tinybird CLI + run: curl -LsSf https://api.tinybird.co/static/install.sh | sh + - name: Build project + run: tb build + - name: Test project + run: tb test run + - name: deploy project + run: tb deployment create --auto --wait + - name: push some data + run: tb datasource append analytics_events datasources/fixtures/analytics_events.ndjson + - name: check data in top_browsers data source + run: tb sql "select count() from top_browsers" + - name: apply patch + run: rm pipes/top_browsers.pipe + - name: deploy project again + run: tb deployment create --auto --wait + - name: check data in top_browsers data source after deploy should fail (because the resource was deleted) + run: tb sql "select count() from top_browsers" || exit 0 && exit 1 + + + drop_column: + runs-on: ubuntu-latest + defaults: + run: + working-directory: './drop_column' + services: + tinybird: + image: tinybirdco/tinybird-local:beta + ports: + - 80:80 + steps: + - uses: actions/checkout@v3 + - name: Set local environment credentials (required for local deployments) + run: | + echo "TB_HOST=http://localhost:80" >> $GITHUB_ENV + echo "TB_TOKEN=$(curl -s http://localhost:80/tokens | jq -r .admin_token)" >> $GITHUB_ENV + - name: Install Tinybird CLI + run: curl -LsSf https://api.tinybird.co/static/install.sh | sh + - name: Build project + run: tb build + - name: Test project + run: tb test run + - name: deploy project + run: tb deployment create --auto --wait + - name: push some data + run: tb datasource append analytics_events datasources/fixtures/analytics_events.ndjson + - name: check data in the landing data source + run: tb sql "select count() from analytics_events" + - name: apply patch + run: cp -r .diff/* . && git diff + - name: deploy project again + run: tb deployment create --auto --wait + - name: check data in the landing data source after deploy + run: tb sql "select count() from analytics_events" + + + ingest_data_in_old_format: + runs-on: ubuntu-latest + defaults: + run: + working-directory: './ingest_data_in_old_format' + services: + tinybird: + image: tinybirdco/tinybird-local:beta + ports: + - 80:80 + steps: + - uses: actions/checkout@v3 + - name: Set local environment credentials (required for local deployments) + run: | + echo "TB_HOST=http://localhost:80" >> $GITHUB_ENV + echo "TB_TOKEN=$(curl -s http://localhost:80/tokens | jq -r .admin_token)" >> $GITHUB_ENV + - name: Install Tinybird CLI + run: curl -LsSf https://api.tinybird.co/static/install.sh | sh + - name: Build project + run: tb build + - name: Test project + run: tb test run + - name: deploy project + run: tb deployment create --auto --wait + - name: push data with the current schema + run: tb datasource append analytics_events datasources/fixtures/analytics_events.ndjson + - name: push data with the new schema (this should fail) + run: tb datasource append analytics_events datasources/fixtures/analytics_events_new.ndjson || exit 0 && exit 1 + - name: check data in the landing data source + run: tb sql "select count() from analytics_events" + - name: apply patch + run: cp -r .diff/* . && git diff + # Deploy the project in two phases: + # 1. Create a new deployment with the new schema + # 2. Push the data with the old schema and check if the data lands properly + # 3. Push the data with the new schema and check if the data lands properly + # 4. Promote the new deployment to the production environment + # + # TODO: In the future, we want to be able to run `tb deployment create --auto --wait` + # and inserts with both schemas and have the data land properly in the landing data source. + - name: deploy project again + run: tb deployment create --wait + - name: push data with the old schema and check if the data lands properly + run: tb datasource append analytics_events datasources/fixtures/analytics_events.ndjson + - name: push data with the new schema and check if the data lands properly + run: tb --preview datasource append analytics_events datasources/fixtures/analytics_events_new.ndjson + - name: check data in the landing after deploy + run: tb sql "select count() from analytics_events" + - name: promote the new deployment to the production environment + run: tb deployment promote --wait + - name: check data in the landing after promote + run: tb sql "select count() from analytics_events" diff --git a/add_new_column_to_a_materialized_view/.diff/datasources/analytics_sessions_mv1.datasource b/add_new_column_to_a_materialized_view/.diff/datasources/analytics_sessions_mv1.datasource new file mode 100644 index 00000000..18f7361a --- /dev/null +++ b/add_new_column_to_a_materialized_view/.diff/datasources/analytics_sessions_mv1.datasource @@ -0,0 +1,15 @@ + +SCHEMA > + `date` Date, + `session_id` String, + `version` SimpleAggregateFunction(any, String), + `device` SimpleAggregateFunction(any, String), + `browser` SimpleAggregateFunction(any, String), + `location` SimpleAggregateFunction(any, String), + `first_hit` SimpleAggregateFunction(min, DateTime), + `latest_hit` SimpleAggregateFunction(max, DateTime), + `hits` AggregateFunction(count) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, session_id" \ No newline at end of file diff --git a/add_new_column_to_a_materialized_view/pipes/analytics_sessions1_backfill.pipe b/add_new_column_to_a_materialized_view/.diff/pipes/analytics_sessions1.pipe similarity index 83% rename from add_new_column_to_a_materialized_view/pipes/analytics_sessions1_backfill.pipe rename to add_new_column_to_a_materialized_view/.diff/pipes/analytics_sessions1.pipe index c2f449e7..07f0b4fb 100644 --- a/add_new_column_to_a_materialized_view/pipes/analytics_sessions1_backfill.pipe +++ b/add_new_column_to_a_materialized_view/.diff/pipes/analytics_sessions1.pipe @@ -1,19 +1,20 @@ -NODE analytics_sessions_1_1_backfill +NODE analytics_sessions_1_1 +DESCRIPTION > + Aggregate by session_id and calculate session metrics SQL > SELECT toDate(timestamp) AS date, session_id, + anySimpleState(version) AS version, anySimpleState(device) AS device, anySimpleState(browser) AS browser, - anySimpleState(version) AS version, anySimpleState(location) AS location, minSimpleState(timestamp) AS first_hit, maxSimpleState(timestamp) AS latest_hit, countState() AS hits FROM analytics_hits - WHERE timestamp <= '2024-05-27 11:00:00' GROUP BY date, session_id diff --git a/add_new_column_to_a_materialized_view/datasources/analytics_sessions_mv1.datasource b/add_new_column_to_a_materialized_view/datasources/analytics_sessions_mv1.datasource index baf591dc..267771df 100644 --- a/add_new_column_to_a_materialized_view/datasources/analytics_sessions_mv1.datasource +++ b/add_new_column_to_a_materialized_view/datasources/analytics_sessions_mv1.datasource @@ -4,7 +4,6 @@ SCHEMA > `session_id` String, `device` SimpleAggregateFunction(any, String), `browser` SimpleAggregateFunction(any, String), - `version` SimpleAggregateFunction(any, String), `location` SimpleAggregateFunction(any, String), `first_hit` SimpleAggregateFunction(min, DateTime), `latest_hit` SimpleAggregateFunction(max, DateTime), diff --git a/change_sorting_key_landing_data_source/datasources/fixtures/analytics_events.ndjson b/add_new_column_to_a_materialized_view/fixtures/analytics_events.ndjson similarity index 84% rename from change_sorting_key_landing_data_source/datasources/fixtures/analytics_events.ndjson rename to add_new_column_to_a_materialized_view/fixtures/analytics_events.ndjson index af3213c6..80646ffa 100644 --- a/change_sorting_key_landing_data_source/datasources/fixtures/analytics_events.ndjson +++ b/add_new_column_to_a_materialized_view/fixtures/analytics_events.ndjson @@ -1,4 +1,4 @@ -{"timestamp":"2023-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} -{"timestamp":"2023-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} -{"timestamp":"2023-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} -{"timestamp":"2023-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2030-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/add_new_column_to_a_materialized_view/pipes/analytics_hits.pipe b/add_new_column_to_a_materialized_view/pipes/analytics_hits.pipe index 1a226c80..f32975c3 100644 --- a/add_new_column_to_a_materialized_view/pipes/analytics_hits.pipe +++ b/add_new_column_to_a_materialized_view/pipes/analytics_hits.pipe @@ -57,3 +57,4 @@ SQL > parsed_hits +TYPE endpoint diff --git a/add_new_column_to_a_materialized_view/pipes/analytics_sessions1.pipe b/add_new_column_to_a_materialized_view/pipes/analytics_sessions1.pipe index 013172a5..20688e67 100644 --- a/add_new_column_to_a_materialized_view/pipes/analytics_sessions1.pipe +++ b/add_new_column_to_a_materialized_view/pipes/analytics_sessions1.pipe @@ -9,13 +9,11 @@ SQL > session_id, anySimpleState(device) AS device, anySimpleState(browser) AS browser, - anySimpleState(version) AS version, anySimpleState(location) AS location, minSimpleState(timestamp) AS first_hit, maxSimpleState(timestamp) AS latest_hit, countState() AS hits FROM analytics_hits - WHERE timestamp > '2024-05-27 11:00:00' GROUP BY date, session_id diff --git a/add_new_column_to_a_materialized_view/pipes/kpis.pipe b/add_new_column_to_a_materialized_view/pipes/kpis.pipe index da28146c..bcce238f 100644 --- a/add_new_column_to_a_materialized_view/pipes/kpis.pipe +++ b/add_new_column_to_a_materialized_view/pipes/kpis.pipe @@ -6,6 +6,8 @@ DESCRIPTION > TOKEN "dashboard" READ + + NODE timeseries DESCRIPTION > Generate a timeseries for the specified time range, so we call fill empty data points. @@ -16,23 +18,35 @@ SQL > % {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} with + {% if defined(date_from) %} toStartOfDay(toDate({{Date(date_from, description="Starting day for filtering a date range", required=False)}})) as start, {% else %} - toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day) as start, + {% else %} + toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} {% end %} + {% if defined(date_to) %} toStartOfDay(toDate({{Date(date_to, description="Finishing day for filtering a date range", required=False)}})) as end {% else %} - toStartOfDay(today()) as end + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}})) as end + {% else %} + toStartOfDay(today()) as end + {% end %} {% end %} + {% if _single_day %} select arrayJoin(arrayMap(x -> toDateTime(x), range(toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600))) as date {% else %} select arrayJoin(arrayMap(x -> toDate(x), range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600))) as date {% end %} + {% if defined(fixed_time) %} - where date <= toDateTime({{DateTime(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + where date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) {% else %} where date <= now() {% end %} @@ -44,7 +58,6 @@ DESCRIPTION > Group by sessions and calculate metrics at that level SQL > - % {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} select @@ -81,7 +94,11 @@ SQL > {% if defined(date_to) %} and date <= {{Date(date_to)}} {% else %} - and date <= today() + {% if defined(fixed_time) %} + and date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + {% else %} + and date <= today() + {% end %} {% end %} group by date, session_id @@ -125,3 +142,5 @@ SQL > left join data b using date + +TYPE endpoint \ No newline at end of file diff --git a/add_new_column_to_a_materialized_view/pipes/top_browsers.pipe b/add_new_column_to_a_materialized_view/pipes/top_browsers.pipe index 4a335814..d0c2d803 100644 --- a/add_new_column_to_a_materialized_view/pipes/top_browsers.pipe +++ b/add_new_column_to_a_materialized_view/pipes/top_browsers.pipe @@ -23,12 +23,20 @@ SQL > {% if defined(date_from) %} date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} {% else %} - date >= timestampAdd(today(), interval -7 day) + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} {% end %} {% if defined(date_to) %} and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} {% else %} - and date <= today() + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} {% end %} group by browser @@ -37,3 +45,4 @@ SQL > limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} +TYPE endpoint diff --git a/add_new_column_to_a_materialized_view/pipes/top_devices.pipe b/add_new_column_to_a_materialized_view/pipes/top_devices.pipe index 2592b5c8..f899f175 100644 --- a/add_new_column_to_a_materialized_view/pipes/top_devices.pipe +++ b/add_new_column_to_a_materialized_view/pipes/top_devices.pipe @@ -36,4 +36,4 @@ SQL > visits desc limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - +TYPE endpoint diff --git a/add_new_column_to_a_materialized_view/pipes/top_locations.pipe b/add_new_column_to_a_materialized_view/pipes/top_locations.pipe index 79018a32..1431e7b8 100644 --- a/add_new_column_to_a_materialized_view/pipes/top_locations.pipe +++ b/add_new_column_to_a_materialized_view/pipes/top_locations.pipe @@ -23,12 +23,20 @@ SQL > {% if defined(date_from) %} date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} {% else %} - date >= timestampAdd(today(), interval -7 day) + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} {% end %} {% if defined(date_to) %} and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} {% else %} - and date <= today() + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} {% end %} group by location @@ -37,3 +45,5 @@ SQL > limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + +TYPE endpoint \ No newline at end of file diff --git a/add_new_column_to_a_materialized_view/pipes/top_pages.pipe b/add_new_column_to_a_materialized_view/pipes/top_pages.pipe index 394f362d..4ade0e7b 100644 --- a/add_new_column_to_a_materialized_view/pipes/top_pages.pipe +++ b/add_new_column_to_a_materialized_view/pipes/top_pages.pipe @@ -23,17 +23,26 @@ SQL > {% if defined(date_from) %} date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} {% else %} - date >= timestampAdd(today(), interval -7 day) + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} {% end %} {% if defined(date_to) %} and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} {% else %} - and date <= today() + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} {% end %} group by pathname order by visits desc limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} +TYPE endpoint diff --git a/add_new_column_to_a_materialized_view/pipes/top_sources.pipe b/add_new_column_to_a_materialized_view/pipes/top_sources.pipe index eb7756ff..0114cc08 100644 --- a/add_new_column_to_a_materialized_view/pipes/top_sources.pipe +++ b/add_new_column_to_a_materialized_view/pipes/top_sources.pipe @@ -36,4 +36,4 @@ SQL > visits desc limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - +TYPE endpoint diff --git a/add_new_column_to_a_materialized_view/pipes/trend.pipe b/add_new_column_to_a_materialized_view/pipes/trend.pipe index 8b1ae36e..4197bcbc 100644 --- a/add_new_column_to_a_materialized_view/pipes/trend.pipe +++ b/add_new_column_to_a_materialized_view/pipes/trend.pipe @@ -58,3 +58,4 @@ SQL > a.t +TYPE endpoint \ No newline at end of file diff --git a/add_new_column_to_a_materialized_view/tests/default.yaml b/add_new_column_to_a_materialized_view/tests/default.yaml deleted file mode 100644 index 27d6376d..00000000 --- a/add_new_column_to_a_materialized_view/tests/default.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- no-visits-without-pageviews: - max_bytes_read: null - max_time: null - sql: | - SELECT * FROM kpis - WHERE visits = 0 AND pageviews > 0 -- no-pageviews-without-visits: - max_bytes_read: null - max_time: null - sql: | - SELECT * FROM kpis - WHERE visits > 0 AND pageviews = 0 - diff --git a/add_new_column_to_a_materialized_view/tests/kpis.yaml b/add_new_column_to_a_materialized_view/tests/kpis.yaml new file mode 100644 index 00000000..ff20e4ef --- /dev/null +++ b/add_new_column_to_a_materialized_view/tests/kpis.yaml @@ -0,0 +1,41 @@ + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-12&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-11&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_last_week + description: Test default behavior showing last 7 days + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_fixed_time + description: Test specific datetime with daily granularity + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} diff --git a/add_new_column_to_a_materialized_view/tests/kpis_default.test b/add_new_column_to_a_materialized_view/tests/kpis_default.test deleted file mode 100644 index 8b258f92..00000000 --- a/add_new_column_to_a_materialized_view/tests/kpis_default.test +++ /dev/null @@ -1 +0,0 @@ -tb pipe data kpis --date_from 2053-10-11 --date_to 2053-10-12 --fixed_time "2053-10-13 00:00:00" --format CSV diff --git a/add_new_column_to_a_materialized_view/tests/kpis_default.test.result b/add_new_column_to_a_materialized_view/tests/kpis_default.test.result deleted file mode 100644 index 953e745d..00000000 --- a/add_new_column_to_a_materialized_view/tests/kpis_default.test.result +++ /dev/null @@ -1,3 +0,0 @@ -"date","visits","pageviews","bounce_rate","avg_session_sec" -"2053-10-11",3,3,1,0 -"2053-10-12",1,1,1,0 diff --git a/add_new_column_to_a_materialized_view/tests/regression.yaml b/add_new_column_to_a_materialized_view/tests/regression.yaml deleted file mode 100644 index 8e4dfa5b..00000000 --- a/add_new_column_to_a_materialized_view/tests/regression.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- pipe: '.*' - tests: - - coverage: - -- pipe: 'analytics_hits' - tests: - - coverage: - config: - # skip until start to use it - skip: True diff --git a/add_new_column_to_a_materialized_view/tests/top_browsers.yaml b/add_new_column_to_a_materialized_view/tests/top_browsers.yaml new file mode 100644 index 00000000..b6909ff4 --- /dev/null +++ b/add_new_column_to_a_materialized_view/tests/top_browsers.yaml @@ -0,0 +1,14 @@ + +- name: top_browsers_date_range + description: Test specific date range for browser statistics + parameters: date_from=2030-10-11&date_to=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} + +- name: top_browsers_default + description: Test top browsers with default parameters (last 7 days) + parameters: fixed_time=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} diff --git a/add_new_column_to_a_materialized_view/tests/top_pages.yaml b/add_new_column_to_a_materialized_view/tests/top_pages.yaml new file mode 100644 index 00000000..099b1523 --- /dev/null +++ b/add_new_column_to_a_materialized_view/tests/top_pages.yaml @@ -0,0 +1,14 @@ + +- name: top_pages_default + description: Test default values (last 7 days, limit 50, skip 0) + parameters: fixed_time=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} + +- name: top_pages_date_range + description: Test specific date range + parameters: date_from=2030-10-11&date_to=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} diff --git a/change_sorting_key_landing_data_source/README.md b/change_sorting_key_landing_data_source/README.md deleted file mode 100644 index ba6a6ce0..00000000 --- a/change_sorting_key_landing_data_source/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Tinybird Versions - Change Sorting Key to a Landing Data Source - -Changing the sorting key of a Data Source requires rewriting the entire table. - -You can opt-in by any of these two options: - -1. Create a Materialized View where the target Data Source has the desired sorting key. -2. Create a new Data Source with the new sorting key, sync the data with the original table and then exchange both tables. - -## 1. New Materialized View - -- Create a new Data Source - -`datasources/analytics_events_sk.datasource` - -```diff - SCHEMA > - `timestamp` DateTime `json:$.timestamp`, - `session_id` String `json:$.session_id`, - `action` LowCardinality(String) `json:$.action`, - `version` LowCardinality(String) `json:$.version`, - `payload` String `json:$.payload` - - ENGINE "MergeTree" -- ENGINE_PARTITION_KEY toYYYYMM(timestamp) -+ ENGINE_PARTITION_KEY toYYYYMMDD(timestamp) - ENGINE_SORTING_KEY timestamp -- ENGINE_SORTING_KEY timestamp, action, version -+ ENGINE_SORTING_KEY "timestamp" - ENGINE_TTL "timestamp + toIntervalDay(60)" -``` - -- Create a new Materialized View `sync_analytics_events.pipe` filtering by a date in the future for backfilling purposes - -``` -NODE sync_analytics_events -SQL > - SELECT * FROM analytics_events - WHERE timestamp > '2024-05-28 00:00:00' - -TYPE MATERIALIZED -DATASOURCE analytics_events_sk -``` - -- Create a Copy Pipe `backfill_analytics_events.pipe` for backfilling. - -``` -NODE backfill_analytics_events -SQL > - % - SELECT * FROM analytics_events - WHERE timestamp BETWEEN {{DateTime(start_backfill_timestamp)}} AND {{DateTime(end_backfill_timestamp)}} -``` - -- Run CI and merge the Pull Request. To run the backfill do the following (you can test it in CI and then run it after merge): - -- Wait for the first event ingested in `analytics_events_sk` -- Run the backfill, you do it with a single command or by chungs: -``` -tb pipe copy run backfill_analytics_events --param start_backfill_timestamp='2024-01-01 00:00:00' --param end_backfill_timestamp='2024-05-28 00:00:00' --wait --yes -``` - -- Cleanup - -Once you finish the backfill you can remove the Copy Pipe and start using the new `analytics_events_sk` in your downstream dependencies. - -## 2. New Data Source with exchange - -Follow the same steps as above, but once the backfill operation over `analytics_events_sk` is finished you can exchange the old and new Data Sources: - -``` -tb datasource exchange analytics_events analytics_events_sk -``` - -The exchange command is useful to replace two tables among them when they have the same exact schema. The exchange command is experimental, contact us at `support@tinybird.co` to enable it on your main Workspace. diff --git a/change_sorting_key_landing_data_source/pipes/kpis.pipe b/change_sorting_key_landing_data_source/pipes/kpis.pipe deleted file mode 100644 index fa85568d..00000000 --- a/change_sorting_key_landing_data_source/pipes/kpis.pipe +++ /dev/null @@ -1,125 +0,0 @@ -DESCRIPTION > - Summary with general KPIs per date, including visits, page views, bounce rate and average session duration. - Accepts `date_from` and `date_to` date filter, all historical data if not passed. - Daily granularity, except when filtering one single day (hourly) - -TOKEN "dashboard" READ - -NODE timeseries -DESCRIPTION > - Generate a timeseries for the specified time range, so we call fill empty data points. - Filters "future" data points. - -SQL > - % - {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} - with - {% if defined(date_from) %} - toStartOfDay( - toDate( - {{ - Date( - date_from, - description="Starting day for filtering a date range", - required=False, - ) - }} - ) - ) as start, - {% else %} toStartOfDay(timestampAdd(today(), interval -7 day)) as start, - {% end %} - {% if defined(date_to) %} - toStartOfDay( - toDate( - {{ - Date( - date_to, - description="Finishing day for filtering a date range", - required=False, - ) - }} - ) - ) as end - {% else %} toStartOfDay(today()) as end - {% end %} - {% if _single_day %} - select - arrayJoin( - arrayMap( - x -> toDateTime(x), - range( - toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600 - ) - ) - ) as date - {% else %} - select - arrayJoin( - arrayMap( - x -> toDate(x), - range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600) - ) - ) as date - {% end %} - where date <= now() - -NODE hits -DESCRIPTION > - Group by sessions and calculate metrics at that level - -SQL > - % - {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} - select - toStartOfHour(timestamp) as date, - session_id, - uniq(session_id) as visits, - count() as pageviews, - case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce, - max(timestamp) as latest_hit_aux, - min(timestamp) as first_hit_aux - from analytics_hits - where toDate(timestamp) = {{ Date(date_from) }} - group by toStartOfHour(timestamp), session_id - {% else %} - select - date, - session_id, - uniq(session_id) as visits, - countMerge(hits) as pageviews, - case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce, - max(latest_hit) as latest_hit_aux, - min(first_hit) as first_hit_aux - from analytics_sessions_mv - where - {% if defined(date_from) %} date >= {{ Date(date_from) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} and date <= {{ Date(date_to) }} - {% else %} and date <= today() - {% end %} - group by date, session_id - {% end %} - -NODE data -DESCRIPTION > - General KPIs per date, works for both summary metrics and trends charts. - -SQL > - select - date, - uniq(session_id) as visits, - sum(pageviews) as pageviews, - sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate, - avg(latest_hit_aux - first_hit_aux) as avg_session_sec - from hits - group by date - -NODE endpoint -DESCRIPTION > - Join and generate timeseries with metrics - -SQL > - select a.date, b.visits, b.pageviews, b.bounce_rate, b.avg_session_sec - from timeseries a - left join data b using date diff --git a/change_sorting_key_landing_data_source/pipes/top_browsers.pipe b/change_sorting_key_landing_data_source/pipes/top_browsers.pipe deleted file mode 100644 index 8e4d136c..00000000 --- a/change_sorting_key_landing_data_source/pipes/top_browsers.pipe +++ /dev/null @@ -1,31 +0,0 @@ -DESCRIPTION > - Top Browsers ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by browser and calcualte hits and visits - -SQL > - % - select browser, uniqMerge(visits) as visits, countMerge(hits) as hits - from analytics_sources_mv - where - {% if defined(date_from) %} - date - >= - {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date - <= - {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} - {% else %} and date <= today() - {% end %} - group by browser - order by visits desc - limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/change_sorting_key_landing_data_source/pipes/top_devices.pipe b/change_sorting_key_landing_data_source/pipes/top_devices.pipe deleted file mode 100644 index 7425f667..00000000 --- a/change_sorting_key_landing_data_source/pipes/top_devices.pipe +++ /dev/null @@ -1,31 +0,0 @@ -DESCRIPTION > - Top Device Types ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by device and calcualte hits and visits - -SQL > - % - select device, uniqMerge(visits) as visits, countMerge(hits) as hits - from analytics_sources_mv - where - {% if defined(date_from) %} - date - >= - {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date - <= - {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} - {% else %} and date <= today() - {% end %} - group by device - order by visits desc - limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/change_sorting_key_landing_data_source/pipes/top_locations.pipe b/change_sorting_key_landing_data_source/pipes/top_locations.pipe deleted file mode 100644 index d6bc1eea..00000000 --- a/change_sorting_key_landing_data_source/pipes/top_locations.pipe +++ /dev/null @@ -1,31 +0,0 @@ -DESCRIPTION > - Top visting Countries ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by pagepath and calcualte hits and visits - -SQL > - % - select location, uniqMerge(visits) as visits, countMerge(hits) as hits - from analytics_pages_mv - where - {% if defined(date_from) %} - date - >= - {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date - <= - {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} - {% else %} and date <= today() - {% end %} - group by location - order by visits desc - limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/change_sorting_key_landing_data_source/pipes/top_pages.pipe b/change_sorting_key_landing_data_source/pipes/top_pages.pipe deleted file mode 100644 index 536e1774..00000000 --- a/change_sorting_key_landing_data_source/pipes/top_pages.pipe +++ /dev/null @@ -1,31 +0,0 @@ -DESCRIPTION > - Most visited pages for a given period. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by pagepath and calcualte hits and visits - -SQL > - % - select pathname, uniqMerge(visits) as visits, countMerge(hits) as hits - from analytics_pages_mv - where - {% if defined(date_from) %} - date - >= - {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date - <= - {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} - {% else %} and date <= today() - {% end %} - group by pathname - order by visits desc - limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/change_sorting_key_landing_data_source/pipes/top_sources.pipe b/change_sorting_key_landing_data_source/pipes/top_sources.pipe deleted file mode 100644 index 5d99901e..00000000 --- a/change_sorting_key_landing_data_source/pipes/top_sources.pipe +++ /dev/null @@ -1,31 +0,0 @@ -DESCRIPTION > - Top traffic sources (domains), ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by referral and calcualte hits and visits - -SQL > - % - select domainWithoutWWW(referrer) as referrer, uniqMerge(visits) as visits, countMerge(hits) as hits - from analytics_sources_mv - where - {% if defined(date_from) %} - date - >= - {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} - {% else %} date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date - <= - {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} - {% else %} and date <= today() - {% end %} - group by referrer - order by visits desc - limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/change_sorting_key_landing_data_source/pipes/trend.pipe b/change_sorting_key_landing_data_source/pipes/trend.pipe deleted file mode 100644 index df84631b..00000000 --- a/change_sorting_key_landing_data_source/pipes/trend.pipe +++ /dev/null @@ -1,32 +0,0 @@ -DESCRIPTION > - Visits trend over time for the last 30 minutes, filling the blanks. - Works great for the realtime chart. - -TOKEN "dashboard" READ - -NODE timeseries -DESCRIPTION > - Generate a timeseries for the last 30 minutes, so we call fill empty data points - -SQL > - with (now() - interval 30 minute) as start - select addMinutes(toStartOfMinute(start), number) as t - from (select arrayJoin(range(1, 31)) as number) - -NODE hits -DESCRIPTION > - Get last 30 minutes metrics gropued by minute - -SQL > - select toStartOfMinute(timestamp) as t, uniq(session_id) as visits - from analytics_hits - where timestamp >= (now() - interval 30 minute) - group by toStartOfMinute(timestamp) - order by toStartOfMinute(timestamp) - -NODE endpoint -DESCRIPTION > - Join and generate timeseries with metrics for the last 30 minutes - -SQL > - select a.t, b.visits from timeseries a left join hits b on a.t = b.t order by a.t diff --git a/change_sorting_key_landing_data_source/scripts/exec_test.sh b/change_sorting_key_landing_data_source/scripts/exec_test.sh deleted file mode 100755 index 3c6b86d2..00000000 --- a/change_sorting_key_landing_data_source/scripts/exec_test.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -export TB_VERSION_WARNING=0 - -run_test() { - t=$1 - echo "** Running $t **" - echo "** $(cat $t)" - tmpfile=$(mktemp) - retries=0 - TOTAL_RETRIES=3 - - # When appending fixtures, we need to retry in case of the data is not replicated in time - while [ $retries -lt $TOTAL_RETRIES ]; do - # Run the test and store the output in a temporary file - bash $t $2 >$tmpfile - exit_code=$? - if [ "$exit_code" -eq 0 ]; then - # If the test passed, break the loop - if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then - break - # If the test failed, increment the retries counter and try again - else - retries=$((retries+1)) - fi - # If the bash command failed, print an error message and break the loop - else - break - fi - done - - if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then - echo "✅ Test $t passed" - rm $tmpfile - return 0 - elif [ $retries -eq $TOTAL_RETRIES ]; then - echo "🚨 ERROR: Test $t failed, diff:"; - diff -B ${t}.result $tmpfile - rm $tmpfile - return 1 - else - echo "🚨 ERROR: Test $t failed with bash command exit code $?" - cat $tmpfile - rm $tmpfile - return 1 - fi - echo "" -} -export -f run_test - -fail=0 -find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1 - -if [ $fail == 1 ]; then - exit -1; -fi diff --git a/change_sorting_key_landing_data_source/tests/default.yaml b/change_sorting_key_landing_data_source/tests/default.yaml deleted file mode 100644 index 27d6376d..00000000 --- a/change_sorting_key_landing_data_source/tests/default.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- no-visits-without-pageviews: - max_bytes_read: null - max_time: null - sql: | - SELECT * FROM kpis - WHERE visits = 0 AND pageviews > 0 -- no-pageviews-without-visits: - max_bytes_read: null - max_time: null - sql: | - SELECT * FROM kpis - WHERE visits > 0 AND pageviews = 0 - diff --git a/change_sorting_key_landing_data_source/tests/kpis_default.test b/change_sorting_key_landing_data_source/tests/kpis_default.test deleted file mode 100644 index 1cf7de30..00000000 --- a/change_sorting_key_landing_data_source/tests/kpis_default.test +++ /dev/null @@ -1 +0,0 @@ -tb pipe data kpis --date_from 2023-10-11 --date_to 2023-10-12 --format CSV diff --git a/change_sorting_key_landing_data_source/tests/kpis_default.test.result b/change_sorting_key_landing_data_source/tests/kpis_default.test.result deleted file mode 100644 index eba69692..00000000 --- a/change_sorting_key_landing_data_source/tests/kpis_default.test.result +++ /dev/null @@ -1,3 +0,0 @@ -"date","visits","pageviews","bounce_rate","avg_session_sec" -"2023-10-11",3,3,1,0 -"2023-10-12",1,1,1,0 diff --git a/change_sorting_key_landing_data_source_2/.diff/datasources/analytics_events.datasource b/change_sorting_key_landing_data_source_2/.diff/datasources/analytics_events.datasource new file mode 100644 index 00000000..342959af --- /dev/null +++ b/change_sorting_key_landing_data_source_2/.diff/datasources/analytics_events.datasource @@ -0,0 +1,16 @@ +TOKEN "tracker" APPEND + +DESCRIPTION > + Analytics events landing data source + +SCHEMA > + `timestamp` DateTime `json:$.timestamp`, + `session_id` String `json:$.session_id`, + `action` LowCardinality(String) `json:$.action`, + `version` LowCardinality(String) `json:$.version`, + `payload` String `json:$.payload` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMMDD(timestamp)" +ENGINE_SORTING_KEY "timestamp, action, version" +ENGINE_TTL "timestamp + toIntervalDay(60)" \ No newline at end of file diff --git a/change_sorting_key_landing_data_source/.tinyenv b/change_sorting_key_landing_data_source_2/.tinyenv similarity index 60% rename from change_sorting_key_landing_data_source/.tinyenv rename to change_sorting_key_landing_data_source_2/.tinyenv index eb93974e..9889abfd 100644 --- a/change_sorting_key_landing_data_source/.tinyenv +++ b/change_sorting_key_landing_data_source_2/.tinyenv @@ -1,2 +1,2 @@ TB_VERSION_WARNING=0 -VERSION=3.0.3 +VERSION=2.0.0 diff --git a/change_sorting_key_landing_data_source_2/README.md b/change_sorting_key_landing_data_source_2/README.md new file mode 100644 index 00000000..5b8c1a69 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/README.md @@ -0,0 +1,72 @@ +# Tinybird Versions - Add Column to a Materialized View + +For that the steps will be: + +1. Create a new Materialized View (Pipe and Data Source) to add the new column. +2. Backfill +3. Connect endpoints to the new Data Source +4. Delete old Pipe and Data Source + +## 1: Create a new Materialized View + +[PR](https://github.com/tinybirdco/use-case-examples/pull/286) + +`analytics_sessions_mv1.datasource`: +```diff +SCHEMA > + `date` Date, + `session_id` String, ++ `version` SimpleAggregateFunction(any, String), + `device` SimpleAggregateFunction(any, String), + `browser` SimpleAggregateFunction(any, String), + `location` SimpleAggregateFunction(any, String), + `first_hit` SimpleAggregateFunction(min, DateTime), + `latest_hit` SimpleAggregateFunction(max, DateTime), + `hits` AggregateFunction(count) +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, session_id" +``` + +`analytics_sessions1.pipe` +```diff +SELECT + toDate(timestamp) AS date, + session_id, ++ anySimpleState(version) AS version, + anySimpleState(device) AS device, + anySimpleState(browser) AS browser, + anySimpleState(location) AS location, + minSimpleState(timestamp) AS first_hit, + maxSimpleState(timestamp) AS latest_hit, + countState() AS hits + FROM analytics_hits + WHERE timestamp > '2024-05-27 11:00:00' + GROUP BY + date, + session_id +``` + +Note we are filtering by a date in the future to be able to perform a backfill operation. + +Additionally we create a Pipe only for backfilling purposes, see `analytics_sessions1_backfill.pipe` in the [PR](https://github.com/tinybirdco/use-case-examples/pull/286) + +## 2: Backfill + +While this can be automated using a custom deployment that runs the backfill operation we recommend it to just do it manually. So once the previous PR has been merged and the backfill DateTime (in this case '2024-05-27 11:00:00') has passed, you can run this command on your main Workspace: + +`tb pipe populate analytics_sessions1_backfill --node analytics_sessions_1_1_backfill --wait` + +Additionally you can run some query with `tb sql` over both Data Sources to check for data quality. + +## 3: Connect endpoints to the new Data Source + +Once you've validated data quality on both Data Sources, you can make the Pipe endpoints to use the new `analytics_sessions_1` Data Source. + +[See PR](https://github.com/tinybirdco/use-case-examples/pull/287) + +## 3: Delete old Pipe and Data Source + +Once the changes are in the main Workspace and running in production you can get rid of old resources by just using `git rm`, create a git branch and merge it. + +[See PR](https://github.com/tinybirdco/use-case-examples/pull/288) diff --git a/change_sorting_key_landing_data_source/datasources/analytics_events.datasource b/change_sorting_key_landing_data_source_2/datasources/analytics_events.datasource similarity index 70% rename from change_sorting_key_landing_data_source/datasources/analytics_events.datasource rename to change_sorting_key_landing_data_source_2/datasources/analytics_events.datasource index 09c755d9..241f6984 100644 --- a/change_sorting_key_landing_data_source/datasources/analytics_events.datasource +++ b/change_sorting_key_landing_data_source_2/datasources/analytics_events.datasource @@ -1,8 +1,8 @@ +TOKEN "tracker" APPEND + DESCRIPTION > Analytics events landing data source -TOKEN "tracker" APPEND - SCHEMA > `timestamp` DateTime `json:$.timestamp`, `session_id` String `json:$.session_id`, @@ -10,7 +10,7 @@ SCHEMA > `version` LowCardinality(String) `json:$.version`, `payload` String `json:$.payload` -ENGINE MergeTree -ENGINE_PARTITION_KEY toYYYYMMDD(timestamp) -ENGINE_SORTING_KEY timestamp -ENGINE_TTL timestamp + toIntervalDay(60) +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" +ENGINE_SORTING_KEY "timestamp" +ENGINE_TTL "timestamp + toIntervalDay(60)" diff --git a/change_sorting_key_landing_data_source/datasources/analytics_pages_mv.datasource b/change_sorting_key_landing_data_source_2/datasources/analytics_pages_mv.datasource similarity index 60% rename from change_sorting_key_landing_data_source/datasources/analytics_pages_mv.datasource rename to change_sorting_key_landing_data_source_2/datasources/analytics_pages_mv.datasource index bc8b5f2c..68a76061 100644 --- a/change_sorting_key_landing_data_source/datasources/analytics_pages_mv.datasource +++ b/change_sorting_key_landing_data_source_2/datasources/analytics_pages_mv.datasource @@ -1,3 +1,4 @@ + SCHEMA > `date` Date, `device` String, @@ -7,6 +8,6 @@ SCHEMA > `visits` AggregateFunction(uniq, String), `hits` AggregateFunction(count) -ENGINE AggregatingMergeTree -ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY date, device, browser, location, pathname +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, device, browser, location, pathname" diff --git a/change_sorting_key_landing_data_source/datasources/analytics_sessions_mv.datasource b/change_sorting_key_landing_data_source_2/datasources/analytics_sessions_mv1.datasource similarity index 76% rename from change_sorting_key_landing_data_source/datasources/analytics_sessions_mv.datasource rename to change_sorting_key_landing_data_source_2/datasources/analytics_sessions_mv1.datasource index 434b98fa..267771df 100644 --- a/change_sorting_key_landing_data_source/datasources/analytics_sessions_mv.datasource +++ b/change_sorting_key_landing_data_source_2/datasources/analytics_sessions_mv1.datasource @@ -1,3 +1,4 @@ + SCHEMA > `date` Date, `session_id` String, @@ -8,6 +9,6 @@ SCHEMA > `latest_hit` SimpleAggregateFunction(max, DateTime), `hits` AggregateFunction(count) -ENGINE AggregatingMergeTree -ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY date, session_id +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, session_id" diff --git a/change_sorting_key_landing_data_source/datasources/analytics_sources_mv.datasource b/change_sorting_key_landing_data_source_2/datasources/analytics_sources_mv.datasource similarity index 60% rename from change_sorting_key_landing_data_source/datasources/analytics_sources_mv.datasource rename to change_sorting_key_landing_data_source_2/datasources/analytics_sources_mv.datasource index b756300c..7235d468 100644 --- a/change_sorting_key_landing_data_source/datasources/analytics_sources_mv.datasource +++ b/change_sorting_key_landing_data_source_2/datasources/analytics_sources_mv.datasource @@ -1,3 +1,4 @@ + SCHEMA > `date` Date, `device` String, @@ -7,6 +8,6 @@ SCHEMA > `visits` AggregateFunction(uniq, String), `hits` AggregateFunction(count) -ENGINE AggregatingMergeTree -ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY date, device, browser, location, referrer +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, device, browser, location, referrer" diff --git a/change_sorting_key_landing_data_source_2/datasources/fixtures/analytics_events.ndjson b/change_sorting_key_landing_data_source_2/datasources/fixtures/analytics_events.ndjson new file mode 100644 index 00000000..a8fa5472 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/datasources/fixtures/analytics_events.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2053-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2053-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/change_sorting_key_landing_data_source_2/fixtures/analytics_events.ndjson b/change_sorting_key_landing_data_source_2/fixtures/analytics_events.ndjson new file mode 100644 index 00000000..80646ffa --- /dev/null +++ b/change_sorting_key_landing_data_source_2/fixtures/analytics_events.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2030-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2030-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/change_sorting_key_landing_data_source_2/pipes/analytics_hits.pipe b/change_sorting_key_landing_data_source_2/pipes/analytics_hits.pipe new file mode 100644 index 00000000..f32975c3 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/analytics_hits.pipe @@ -0,0 +1,60 @@ +DESCRIPTION > + Parsed `page_hit` events, implementing `browser` and `device` detection logic. + + +TOKEN "dashboard" READ + +NODE parsed_hits +DESCRIPTION > + Parse raw page_hit events + +SQL > + + SELECT + timestamp, + action, + version, + coalesce(session_id,'0') as session_id, + JSONExtractString(payload, 'locale') as locale, + JSONExtractString(payload, 'location') as location, + JSONExtractString(payload, 'referrer') as referrer, + JSONExtractString(payload, 'pathname') as pathname, + JSONExtractString(payload, 'href') as href, + lower(JSONExtractString(payload, 'user-agent')) as user_agent + FROM + analytics_events + where action = 'page_hit' + + + +NODE endpoint +SQL > + + SELECT + timestamp, + action, + version, + session_id, + location, + referrer, + pathname, + href, + case + when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') then 'bot' + when match(user_agent, 'android') then 'mobile-android' + when match(user_agent, 'ipad|iphone|ipod') then 'mobile-ios' + else 'desktop' + END as device, + case + when match(user_agent, 'firefox') then 'firefox' + when match(user_agent, 'chrome|crios') then 'chrome' + when match(user_agent, 'opera') then 'opera' + when match(user_agent, 'msie|trident') then 'ie' + when match(user_agent, 'iphone|ipad|safari') then 'safari' + else 'Unknown' + END as browser + FROM + parsed_hits + + +TYPE endpoint diff --git a/change_sorting_key_landing_data_source/pipes/analytics_pages.pipe b/change_sorting_key_landing_data_source_2/pipes/analytics_pages.pipe similarity index 74% rename from change_sorting_key_landing_data_source/pipes/analytics_pages.pipe rename to change_sorting_key_landing_data_source_2/pipes/analytics_pages.pipe index aab9ae50..51e8fb4e 100644 --- a/change_sorting_key_landing_data_source/pipes/analytics_pages.pipe +++ b/change_sorting_key_landing_data_source_2/pipes/analytics_pages.pipe @@ -3,6 +3,7 @@ DESCRIPTION > Aggregate by pathname and calculate session and hits SQL > + SELECT toDate(timestamp) AS date, device, @@ -12,7 +13,14 @@ SQL > uniqState(session_id) AS visits, countState() AS hits FROM analytics_hits - GROUP BY date, device, browser, location, pathname + GROUP BY + date, + device, + browser, + location, + pathname -TYPE MATERIALIZED +TYPE materialized DATASOURCE analytics_pages_mv + + diff --git a/change_sorting_key_landing_data_source/pipes/analytics_sessions.pipe b/change_sorting_key_landing_data_source_2/pipes/analytics_sessions1.pipe similarity index 77% rename from change_sorting_key_landing_data_source/pipes/analytics_sessions.pipe rename to change_sorting_key_landing_data_source_2/pipes/analytics_sessions1.pipe index 3cfd034b..20688e67 100644 --- a/change_sorting_key_landing_data_source/pipes/analytics_sessions.pipe +++ b/change_sorting_key_landing_data_source_2/pipes/analytics_sessions1.pipe @@ -1,8 +1,9 @@ -NODE analytics_sessions_1 +NODE analytics_sessions_1_1 DESCRIPTION > Aggregate by session_id and calculate session metrics SQL > + SELECT toDate(timestamp) AS date, session_id, @@ -13,7 +14,9 @@ SQL > maxSimpleState(timestamp) AS latest_hit, countState() AS hits FROM analytics_hits - GROUP BY date, session_id + GROUP BY + date, + session_id -TYPE MATERIALIZED -DATASOURCE analytics_sessions_mv +TYPE materialized +DATASOURCE analytics_sessions_mv1 diff --git a/change_sorting_key_landing_data_source/pipes/analytics_sources.pipe b/change_sorting_key_landing_data_source_2/pipes/analytics_sources.pipe similarity index 81% rename from change_sorting_key_landing_data_source/pipes/analytics_sources.pipe rename to change_sorting_key_landing_data_source_2/pipes/analytics_sources.pipe index 813519ef..21972934 100644 --- a/change_sorting_key_landing_data_source/pipes/analytics_sources.pipe +++ b/change_sorting_key_landing_data_source_2/pipes/analytics_sources.pipe @@ -3,8 +3,8 @@ DESCRIPTION > Aggregate by referral and calculate session and hits SQL > - WITH - ( + + WITH ( SELECT domainWithoutWWW(href) FROM analytics_hits WHERE href is not null and href != '' @@ -20,7 +20,14 @@ SQL > countState() AS hits FROM analytics_hits WHERE domainWithoutWWW(referrer) != currenct_domain - GROUP BY date, device, browser, location, referrer + GROUP BY + date, + device, + browser, + location, + referrer -TYPE MATERIALIZED +TYPE materialized DATASOURCE analytics_sources_mv + + diff --git a/change_sorting_key_landing_data_source_2/pipes/kpis.pipe b/change_sorting_key_landing_data_source_2/pipes/kpis.pipe new file mode 100644 index 00000000..bcce238f --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/kpis.pipe @@ -0,0 +1,146 @@ +DESCRIPTION > + Summary with general KPIs per date, including visits, page views, bounce rate and average session duration. + Accepts `date_from` and `date_to` date filter, all historical data if not passed. + Daily granularity, except when filtering one single day (hourly) + + +TOKEN "dashboard" READ + + + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the specified time range, so we call fill empty data points. + Filters "future" data points. + +SQL > + + % + {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} + with + + {% if defined(date_from) %} + toStartOfDay(toDate({{Date(date_from, description="Starting day for filtering a date range", required=False)}})) as start, + {% else %} + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day) as start, + {% else %} + toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} + {% end %} + + {% if defined(date_to) %} + toStartOfDay(toDate({{Date(date_to, description="Finishing day for filtering a date range", required=False)}})) as end + {% else %} + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}})) as end + {% else %} + toStartOfDay(today()) as end + {% end %} + {% end %} + + {% if _single_day %} + select arrayJoin(arrayMap(x -> toDateTime(x), range(toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600))) as date + {% else %} + select arrayJoin(arrayMap(x -> toDate(x), range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600))) as date + {% end %} + + {% if defined(fixed_time) %} + where date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + {% else %} + where date <= now() + {% end %} + + + +NODE hits +DESCRIPTION > + Group by sessions and calculate metrics at that level + +SQL > + % + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + select + toStartOfHour(timestamp) as date, + session_id, + uniq(session_id) as visits, + count() as pageviews, + case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce, + max(timestamp) as latest_hit_aux, + min(timestamp) as first_hit_aux + from + analytics_hits + where + toDate(timestamp) = {{Date(date_from)}} + group by + toStartOfHour(timestamp), session_id + {% else %} + select + date, + session_id, + uniq(session_id) as visits, + countMerge(hits) as pageviews, + case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce, + max(latest_hit) as latest_hit_aux, + min(first_hit) as first_hit_aux + from + analytics_sessions_mv1 + where + {% if defined(date_from) %} + date >= {{Date(date_from)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to)}} + {% else %} + {% if defined(fixed_time) %} + and date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + {% else %} + and date <= today() + {% end %} + {% end %} + group by + date, session_id + {% end %} + + + +NODE data +DESCRIPTION > + General KPIs per date, works for both summary metrics and trends charts. + +SQL > + + select + date, + uniq(session_id) as visits, + sum(pageviews) as pageviews, + sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate, + avg(latest_hit_aux - first_hit_aux) as avg_session_sec + from + hits + group by + date + + + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics + +SQL > + + select + a.date, + b.visits, + b.pageviews, + b.bounce_rate, + b.avg_session_sec + from + timeseries a + left join data b using date + + + +TYPE endpoint \ No newline at end of file diff --git a/change_sorting_key_landing_data_source_2/pipes/top_browsers.pipe b/change_sorting_key_landing_data_source_2/pipes/top_browsers.pipe new file mode 100644 index 00000000..d0c2d803 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/top_browsers.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + Top Browsers ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by browser and calcualte hits and visits + +SQL > + + % + select + browser, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + browser + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + + +TYPE endpoint diff --git a/change_sorting_key_landing_data_source_2/pipes/top_devices.pipe b/change_sorting_key_landing_data_source_2/pipes/top_devices.pipe new file mode 100644 index 00000000..f899f175 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/top_devices.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Top Device Types ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by device and calcualte hits and visits + +SQL > + + % + select + device, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + device + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + +TYPE endpoint diff --git a/change_sorting_key_landing_data_source_2/pipes/top_locations.pipe b/change_sorting_key_landing_data_source_2/pipes/top_locations.pipe new file mode 100644 index 00000000..1431e7b8 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/top_locations.pipe @@ -0,0 +1,49 @@ +DESCRIPTION > + Top visting Countries ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calcualte hits and visits + +SQL > + + % + select + location, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + location + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + + + +TYPE endpoint \ No newline at end of file diff --git a/change_sorting_key_landing_data_source_2/pipes/top_pages.pipe b/change_sorting_key_landing_data_source_2/pipes/top_pages.pipe new file mode 100644 index 00000000..4ade0e7b --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/top_pages.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + Most visited pages for a given period. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calcualte hits and visits + +SQL > + + % + select + pathname, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + pathname + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} +TYPE endpoint + + diff --git a/change_sorting_key_landing_data_source_2/pipes/top_sources.pipe b/change_sorting_key_landing_data_source_2/pipes/top_sources.pipe new file mode 100644 index 00000000..0114cc08 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/top_sources.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Top traffic sources (domains), ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by referral and calcualte hits and visits + +SQL > + + % + select + domainWithoutWWW(referrer) as referrer, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + referrer + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + +TYPE endpoint diff --git a/change_sorting_key_landing_data_source_2/pipes/trend.pipe b/change_sorting_key_landing_data_source_2/pipes/trend.pipe new file mode 100644 index 00000000..4197bcbc --- /dev/null +++ b/change_sorting_key_landing_data_source_2/pipes/trend.pipe @@ -0,0 +1,61 @@ +DESCRIPTION > + Visits trend over time for the last 30 minutes, filling the blanks. + Works great for the realtime chart. + + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the last 30 minutes, so we call fill empty data points + +SQL > + + with (now() - interval 30 minute) as start + select addMinutes(toStartOfMinute(start), number) as t + from ( + select arrayJoin(range(1, 31)) as number + ) + + + +NODE hits +DESCRIPTION > + Get last 30 minutes metrics gropued by minute + +SQL > + + select + toStartOfMinute(timestamp) as t, + uniq(session_id) as visits + from + analytics_hits + where + timestamp >= (now() - interval 30 minute) + group by + toStartOfMinute(timestamp) + order by + toStartOfMinute(timestamp) + + + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics for the last 30 minutes + +SQL > + + select + a.t, + b.visits + from + timeseries a + left join + hits b + on + a.t = b.t + order by + a.t + + +TYPE endpoint \ No newline at end of file diff --git a/change_sorting_key_landing_data_source/requirements.txt b/change_sorting_key_landing_data_source_2/requirements.txt similarity index 100% rename from change_sorting_key_landing_data_source/requirements.txt rename to change_sorting_key_landing_data_source_2/requirements.txt diff --git a/change_sorting_key_landing_data_source/scripts/append_fixtures.sh b/change_sorting_key_landing_data_source_2/scripts/append_fixtures.sh similarity index 100% rename from change_sorting_key_landing_data_source/scripts/append_fixtures.sh rename to change_sorting_key_landing_data_source_2/scripts/append_fixtures.sh diff --git a/change_sorting_key_landing_data_source_2/scripts/exec_test.sh b/change_sorting_key_landing_data_source_2/scripts/exec_test.sh new file mode 100755 index 00000000..8b018209 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/scripts/exec_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +export TB_VERSION_WARNING=0 + +run_test() { + t=$1 + echo "** Running $t **" + echo "** $(cat $t)" + if res=$(bash $t $2 | diff -B ${t}.result -); then + echo 'OK'; + else + echo "failed, diff:"; + echo "$res"; + return 1 + fi + echo "" +} +export -f run_test + +fail=0 +find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1 + +if [ $fail == 1 ]; then + exit -1; +fi diff --git a/change_sorting_key_landing_data_source_2/tests/kpis.yaml b/change_sorting_key_landing_data_source_2/tests/kpis.yaml new file mode 100644 index 00000000..ff20e4ef --- /dev/null +++ b/change_sorting_key_landing_data_source_2/tests/kpis.yaml @@ -0,0 +1,41 @@ + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-12&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-11&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_last_week + description: Test default behavior showing last 7 days + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_fixed_time + description: Test specific datetime with daily granularity + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} diff --git a/change_sorting_key_landing_data_source_2/tests/top_browsers.yaml b/change_sorting_key_landing_data_source_2/tests/top_browsers.yaml new file mode 100644 index 00000000..b6909ff4 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/tests/top_browsers.yaml @@ -0,0 +1,14 @@ + +- name: top_browsers_date_range + description: Test specific date range for browser statistics + parameters: date_from=2030-10-11&date_to=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} + +- name: top_browsers_default + description: Test top browsers with default parameters (last 7 days) + parameters: fixed_time=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} diff --git a/change_sorting_key_landing_data_source_2/tests/top_pages.yaml b/change_sorting_key_landing_data_source_2/tests/top_pages.yaml new file mode 100644 index 00000000..099b1523 --- /dev/null +++ b/change_sorting_key_landing_data_source_2/tests/top_pages.yaml @@ -0,0 +1,14 @@ + +- name: top_pages_default + description: Test default values (last 7 days, limit 50, skip 0) + parameters: fixed_time=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} + +- name: top_pages_date_range + description: Test specific date range + parameters: date_from=2030-10-11&date_to=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} diff --git a/delete_simple_resource/pipes/analytics_hits.pipe b/delete_simple_resource/pipes/analytics_hits.pipe index 1a226c80..5b8814dd 100644 --- a/delete_simple_resource/pipes/analytics_hits.pipe +++ b/delete_simple_resource/pipes/analytics_hits.pipe @@ -56,4 +56,4 @@ SQL > FROM parsed_hits - +TYPE endpoint diff --git a/delete_simple_resource/pipes/top_browsers.pipe b/delete_simple_resource/pipes/top_browsers.pipe new file mode 100644 index 00000000..395f5370 --- /dev/null +++ b/delete_simple_resource/pipes/top_browsers.pipe @@ -0,0 +1,35 @@ +DESCRIPTION > + Top Browsers ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by browser and calcualte hits and visits + +SQL > + + % + select browser, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by browser + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE endpoint diff --git a/drop_column/.diff/datasources/analytics_events.datasource b/drop_column/.diff/datasources/analytics_events.datasource new file mode 100644 index 00000000..9fa11921 --- /dev/null +++ b/drop_column/.diff/datasources/analytics_events.datasource @@ -0,0 +1,18 @@ +TOKEN "tracker" APPEND + +DESCRIPTION > + Analytics events landing data source + +SCHEMA > + `timestamp` DateTime `json:$.timestamp`, + `session_id` String `json:$.session_id`, + `action` LowCardinality(String) `json:$.action`, + `payload` String `json:$.payload` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" +ENGINE_SORTING_KEY "timestamp" +ENGINE_TTL "timestamp + toIntervalDay(60)" + +FORWARD_QUERY > + SELECT timestamp, action, session_id, payload diff --git a/change_sorting_key_landing_data_source/pipes/analytics_hits.pipe b/drop_column/.diff/pipes/analytics_hits.pipe similarity index 93% rename from change_sorting_key_landing_data_source/pipes/analytics_hits.pipe rename to drop_column/.diff/pipes/analytics_hits.pipe index e237656b..c2e3d087 100644 --- a/change_sorting_key_landing_data_source/pipes/analytics_hits.pipe +++ b/drop_column/.diff/pipes/analytics_hits.pipe @@ -1,5 +1,6 @@ DESCRIPTION > - Parsed `page_hit` events, implementing `browser` and `device` detection logic. + Parsed `page_hit` events, implementing `browser` and `device` detection logic. + TOKEN "dashboard" READ @@ -8,10 +9,10 @@ DESCRIPTION > Parse raw page_hit events SQL > + SELECT timestamp, action, - version, coalesce(session_id, '0') as session_id, JSONExtractString(payload, 'locale') as locale, JSONExtractString(payload, 'location') as location, @@ -22,12 +23,14 @@ SQL > FROM analytics_events where action = 'page_hit' + + NODE endpoint SQL > + SELECT timestamp, action, - version, session_id, location, referrer, @@ -56,3 +59,6 @@ SQL > else 'Unknown' END as browser FROM parsed_hits + + +TYPE endpoint diff --git a/drop_column/datasources/analytics_events.datasource b/drop_column/datasources/analytics_events.datasource index 7f1cf255..241f6984 100644 --- a/drop_column/datasources/analytics_events.datasource +++ b/drop_column/datasources/analytics_events.datasource @@ -7,6 +7,7 @@ SCHEMA > `timestamp` DateTime `json:$.timestamp`, `session_id` String `json:$.session_id`, `action` LowCardinality(String) `json:$.action`, + `version` LowCardinality(String) `json:$.version`, `payload` String `json:$.payload` ENGINE "MergeTree" diff --git a/drop_column/datasources/fixtures/analytics_events.ndjson b/drop_column/datasources/fixtures/analytics_events.ndjson new file mode 100644 index 00000000..80646ffa --- /dev/null +++ b/drop_column/datasources/fixtures/analytics_events.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2030-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2030-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/drop_column/pipes/analytics_hits.pipe b/drop_column/pipes/analytics_hits.pipe index 05400081..b68d0e64 100644 --- a/drop_column/pipes/analytics_hits.pipe +++ b/drop_column/pipes/analytics_hits.pipe @@ -63,3 +63,4 @@ SQL > FROM parsed_hits +TYPE endpoint diff --git a/drop_column/tests/.gitkeep b/drop_column/tests/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/drop_column/tests/kpis.yaml b/drop_column/tests/kpis.yaml new file mode 100644 index 00000000..ff20e4ef --- /dev/null +++ b/drop_column/tests/kpis.yaml @@ -0,0 +1,41 @@ + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-12&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-11&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_last_week + description: Test default behavior showing last 7 days + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_fixed_time + description: Test specific datetime with daily granularity + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} diff --git a/drop_column/tests/top_browsers.yaml b/drop_column/tests/top_browsers.yaml new file mode 100644 index 00000000..b6909ff4 --- /dev/null +++ b/drop_column/tests/top_browsers.yaml @@ -0,0 +1,14 @@ + +- name: top_browsers_date_range + description: Test specific date range for browser statistics + parameters: date_from=2030-10-11&date_to=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} + +- name: top_browsers_default + description: Test top browsers with default parameters (last 7 days) + parameters: fixed_time=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} diff --git a/drop_column/tests/top_pages.yaml b/drop_column/tests/top_pages.yaml new file mode 100644 index 00000000..099b1523 --- /dev/null +++ b/drop_column/tests/top_pages.yaml @@ -0,0 +1,14 @@ + +- name: top_pages_default + description: Test default values (last 7 days, limit 50, skip 0) + parameters: fixed_time=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} + +- name: top_pages_date_range + description: Test specific date range + parameters: date_from=2030-10-11&date_to=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} diff --git a/ingest_data_in_old_format/.diff/datasources/analytics_events.datasource b/ingest_data_in_old_format/.diff/datasources/analytics_events.datasource new file mode 100644 index 00000000..73c7ea2b --- /dev/null +++ b/ingest_data_in_old_format/.diff/datasources/analytics_events.datasource @@ -0,0 +1,19 @@ +TOKEN "tracker" APPEND + +DESCRIPTION > + Analytics events landing data source + +SCHEMA > + `timestamp` DateTime `json:$.timestamp`, + `session_id` String `json:$.session_id`, + `action` LowCardinality(String) `json:$.action`, + `version` Int32 `json:$.version`, + `payload` String `json:$.payload` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMMDD(timestamp)" +ENGINE_SORTING_KEY "timestamp, action, version" +ENGINE_TTL "timestamp + toIntervalDay(60)" + +FORWARD_QUERY > + select timestamp, session_id, action, cast(version, 'Int32') AS version, payload diff --git a/ingest_data_in_old_format/.tinyenv b/ingest_data_in_old_format/.tinyenv new file mode 100644 index 00000000..9889abfd --- /dev/null +++ b/ingest_data_in_old_format/.tinyenv @@ -0,0 +1,2 @@ +TB_VERSION_WARNING=0 +VERSION=2.0.0 diff --git a/ingest_data_in_old_format/README.md b/ingest_data_in_old_format/README.md new file mode 100644 index 00000000..a43d3243 --- /dev/null +++ b/ingest_data_in_old_format/README.md @@ -0,0 +1,3 @@ +ingest_data_in_old_format + +checks if data lands properly in a datasource after changing the schema \ No newline at end of file diff --git a/ingest_data_in_old_format/datasources/analytics_events.datasource b/ingest_data_in_old_format/datasources/analytics_events.datasource new file mode 100644 index 00000000..241f6984 --- /dev/null +++ b/ingest_data_in_old_format/datasources/analytics_events.datasource @@ -0,0 +1,16 @@ +TOKEN "tracker" APPEND + +DESCRIPTION > + Analytics events landing data source + +SCHEMA > + `timestamp` DateTime `json:$.timestamp`, + `session_id` String `json:$.session_id`, + `action` LowCardinality(String) `json:$.action`, + `version` LowCardinality(String) `json:$.version`, + `payload` String `json:$.payload` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" +ENGINE_SORTING_KEY "timestamp" +ENGINE_TTL "timestamp + toIntervalDay(60)" diff --git a/ingest_data_in_old_format/datasources/analytics_pages_mv.datasource b/ingest_data_in_old_format/datasources/analytics_pages_mv.datasource new file mode 100644 index 00000000..68a76061 --- /dev/null +++ b/ingest_data_in_old_format/datasources/analytics_pages_mv.datasource @@ -0,0 +1,13 @@ + +SCHEMA > + `date` Date, + `device` String, + `browser` String, + `location` String, + `pathname` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, device, browser, location, pathname" diff --git a/ingest_data_in_old_format/datasources/analytics_sessions_mv1.datasource b/ingest_data_in_old_format/datasources/analytics_sessions_mv1.datasource new file mode 100644 index 00000000..267771df --- /dev/null +++ b/ingest_data_in_old_format/datasources/analytics_sessions_mv1.datasource @@ -0,0 +1,14 @@ + +SCHEMA > + `date` Date, + `session_id` String, + `device` SimpleAggregateFunction(any, String), + `browser` SimpleAggregateFunction(any, String), + `location` SimpleAggregateFunction(any, String), + `first_hit` SimpleAggregateFunction(min, DateTime), + `latest_hit` SimpleAggregateFunction(max, DateTime), + `hits` AggregateFunction(count) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, session_id" diff --git a/ingest_data_in_old_format/datasources/analytics_sources_mv.datasource b/ingest_data_in_old_format/datasources/analytics_sources_mv.datasource new file mode 100644 index 00000000..7235d468 --- /dev/null +++ b/ingest_data_in_old_format/datasources/analytics_sources_mv.datasource @@ -0,0 +1,13 @@ + +SCHEMA > + `date` Date, + `device` String, + `browser` String, + `location` String, + `referrer` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, device, browser, location, referrer" diff --git a/ingest_data_in_old_format/datasources/fixtures/analytics_events.ndjson b/ingest_data_in_old_format/datasources/fixtures/analytics_events.ndjson new file mode 100644 index 00000000..a8fa5472 --- /dev/null +++ b/ingest_data_in_old_format/datasources/fixtures/analytics_events.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2053-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2053-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/ingest_data_in_old_format/datasources/fixtures/analytics_events_new.ndjson b/ingest_data_in_old_format/datasources/fixtures/analytics_events_new.ndjson new file mode 100644 index 00000000..3d3615f5 --- /dev/null +++ b/ingest_data_in_old_format/datasources/fixtures/analytics_events_new.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2053-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":1,"payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":1,"payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2053-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":1,"payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2053-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":1,"payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/ingest_data_in_old_format/fixtures/analytics_events.ndjson b/ingest_data_in_old_format/fixtures/analytics_events.ndjson new file mode 100644 index 00000000..80646ffa --- /dev/null +++ b/ingest_data_in_old_format/fixtures/analytics_events.ndjson @@ -0,0 +1,4 @@ +{"timestamp":"2030-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"} +{"timestamp":"2030-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} +{"timestamp":"2030-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"} diff --git a/ingest_data_in_old_format/pipes/analytics_hits.pipe b/ingest_data_in_old_format/pipes/analytics_hits.pipe new file mode 100644 index 00000000..f32975c3 --- /dev/null +++ b/ingest_data_in_old_format/pipes/analytics_hits.pipe @@ -0,0 +1,60 @@ +DESCRIPTION > + Parsed `page_hit` events, implementing `browser` and `device` detection logic. + + +TOKEN "dashboard" READ + +NODE parsed_hits +DESCRIPTION > + Parse raw page_hit events + +SQL > + + SELECT + timestamp, + action, + version, + coalesce(session_id,'0') as session_id, + JSONExtractString(payload, 'locale') as locale, + JSONExtractString(payload, 'location') as location, + JSONExtractString(payload, 'referrer') as referrer, + JSONExtractString(payload, 'pathname') as pathname, + JSONExtractString(payload, 'href') as href, + lower(JSONExtractString(payload, 'user-agent')) as user_agent + FROM + analytics_events + where action = 'page_hit' + + + +NODE endpoint +SQL > + + SELECT + timestamp, + action, + version, + session_id, + location, + referrer, + pathname, + href, + case + when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') then 'bot' + when match(user_agent, 'android') then 'mobile-android' + when match(user_agent, 'ipad|iphone|ipod') then 'mobile-ios' + else 'desktop' + END as device, + case + when match(user_agent, 'firefox') then 'firefox' + when match(user_agent, 'chrome|crios') then 'chrome' + when match(user_agent, 'opera') then 'opera' + when match(user_agent, 'msie|trident') then 'ie' + when match(user_agent, 'iphone|ipad|safari') then 'safari' + else 'Unknown' + END as browser + FROM + parsed_hits + + +TYPE endpoint diff --git a/ingest_data_in_old_format/pipes/analytics_pages.pipe b/ingest_data_in_old_format/pipes/analytics_pages.pipe new file mode 100644 index 00000000..51e8fb4e --- /dev/null +++ b/ingest_data_in_old_format/pipes/analytics_pages.pipe @@ -0,0 +1,26 @@ +NODE analytics_pages_1 +DESCRIPTION > + Aggregate by pathname and calculate session and hits + +SQL > + + SELECT + toDate(timestamp) AS date, + device, + browser, + location, + pathname, + uniqState(session_id) AS visits, + countState() AS hits + FROM analytics_hits + GROUP BY + date, + device, + browser, + location, + pathname + +TYPE materialized +DATASOURCE analytics_pages_mv + + diff --git a/ingest_data_in_old_format/pipes/analytics_sessions1.pipe b/ingest_data_in_old_format/pipes/analytics_sessions1.pipe new file mode 100644 index 00000000..20688e67 --- /dev/null +++ b/ingest_data_in_old_format/pipes/analytics_sessions1.pipe @@ -0,0 +1,22 @@ +NODE analytics_sessions_1_1 +DESCRIPTION > + Aggregate by session_id and calculate session metrics + +SQL > + + SELECT + toDate(timestamp) AS date, + session_id, + anySimpleState(device) AS device, + anySimpleState(browser) AS browser, + anySimpleState(location) AS location, + minSimpleState(timestamp) AS first_hit, + maxSimpleState(timestamp) AS latest_hit, + countState() AS hits + FROM analytics_hits + GROUP BY + date, + session_id + +TYPE materialized +DATASOURCE analytics_sessions_mv1 diff --git a/ingest_data_in_old_format/pipes/analytics_sources.pipe b/ingest_data_in_old_format/pipes/analytics_sources.pipe new file mode 100644 index 00000000..21972934 --- /dev/null +++ b/ingest_data_in_old_format/pipes/analytics_sources.pipe @@ -0,0 +1,33 @@ +NODE analytics_sources_1 +DESCRIPTION > + Aggregate by referral and calculate session and hits + +SQL > + + WITH ( + SELECT domainWithoutWWW(href) + FROM analytics_hits + WHERE href is not null and href != '' + LIMIT 1 + ) AS currenct_domain + SELECT + toDate(timestamp) AS date, + device, + browser, + location, + referrer, + uniqState(session_id) AS visits, + countState() AS hits + FROM analytics_hits + WHERE domainWithoutWWW(referrer) != currenct_domain + GROUP BY + date, + device, + browser, + location, + referrer + +TYPE materialized +DATASOURCE analytics_sources_mv + + diff --git a/ingest_data_in_old_format/pipes/kpis.pipe b/ingest_data_in_old_format/pipes/kpis.pipe new file mode 100644 index 00000000..bcce238f --- /dev/null +++ b/ingest_data_in_old_format/pipes/kpis.pipe @@ -0,0 +1,146 @@ +DESCRIPTION > + Summary with general KPIs per date, including visits, page views, bounce rate and average session duration. + Accepts `date_from` and `date_to` date filter, all historical data if not passed. + Daily granularity, except when filtering one single day (hourly) + + +TOKEN "dashboard" READ + + + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the specified time range, so we call fill empty data points. + Filters "future" data points. + +SQL > + + % + {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} + with + + {% if defined(date_from) %} + toStartOfDay(toDate({{Date(date_from, description="Starting day for filtering a date range", required=False)}})) as start, + {% else %} + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day) as start, + {% else %} + toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} + {% end %} + + {% if defined(date_to) %} + toStartOfDay(toDate({{Date(date_to, description="Finishing day for filtering a date range", required=False)}})) as end + {% else %} + {% if defined(fixed_time) %} + toStartOfDay(toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}})) as end + {% else %} + toStartOfDay(today()) as end + {% end %} + {% end %} + + {% if _single_day %} + select arrayJoin(arrayMap(x -> toDateTime(x), range(toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600))) as date + {% else %} + select arrayJoin(arrayMap(x -> toDate(x), range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600))) as date + {% end %} + + {% if defined(fixed_time) %} + where date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + {% else %} + where date <= now() + {% end %} + + + +NODE hits +DESCRIPTION > + Group by sessions and calculate metrics at that level + +SQL > + % + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + select + toStartOfHour(timestamp) as date, + session_id, + uniq(session_id) as visits, + count() as pageviews, + case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce, + max(timestamp) as latest_hit_aux, + min(timestamp) as first_hit_aux + from + analytics_hits + where + toDate(timestamp) = {{Date(date_from)}} + group by + toStartOfHour(timestamp), session_id + {% else %} + select + date, + session_id, + uniq(session_id) as visits, + countMerge(hits) as pageviews, + case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce, + max(latest_hit) as latest_hit_aux, + min(first_hit) as first_hit_aux + from + analytics_sessions_mv1 + where + {% if defined(date_from) %} + date >= {{Date(date_from)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to)}} + {% else %} + {% if defined(fixed_time) %} + and date <= toDateTime({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) + {% else %} + and date <= today() + {% end %} + {% end %} + group by + date, session_id + {% end %} + + + +NODE data +DESCRIPTION > + General KPIs per date, works for both summary metrics and trends charts. + +SQL > + + select + date, + uniq(session_id) as visits, + sum(pageviews) as pageviews, + sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate, + avg(latest_hit_aux - first_hit_aux) as avg_session_sec + from + hits + group by + date + + + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics + +SQL > + + select + a.date, + b.visits, + b.pageviews, + b.bounce_rate, + b.avg_session_sec + from + timeseries a + left join data b using date + + + +TYPE endpoint \ No newline at end of file diff --git a/ingest_data_in_old_format/pipes/top_browsers.pipe b/ingest_data_in_old_format/pipes/top_browsers.pipe new file mode 100644 index 00000000..d0c2d803 --- /dev/null +++ b/ingest_data_in_old_format/pipes/top_browsers.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + Top Browsers ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by browser and calcualte hits and visits + +SQL > + + % + select + browser, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + browser + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + + +TYPE endpoint diff --git a/ingest_data_in_old_format/pipes/top_devices.pipe b/ingest_data_in_old_format/pipes/top_devices.pipe new file mode 100644 index 00000000..f899f175 --- /dev/null +++ b/ingest_data_in_old_format/pipes/top_devices.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Top Device Types ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by device and calcualte hits and visits + +SQL > + + % + select + device, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + device + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + +TYPE endpoint diff --git a/ingest_data_in_old_format/pipes/top_locations.pipe b/ingest_data_in_old_format/pipes/top_locations.pipe new file mode 100644 index 00000000..1431e7b8 --- /dev/null +++ b/ingest_data_in_old_format/pipes/top_locations.pipe @@ -0,0 +1,49 @@ +DESCRIPTION > + Top visting Countries ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calcualte hits and visits + +SQL > + + % + select + location, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + location + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + + + +TYPE endpoint \ No newline at end of file diff --git a/ingest_data_in_old_format/pipes/top_pages.pipe b/ingest_data_in_old_format/pipes/top_pages.pipe new file mode 100644 index 00000000..4ade0e7b --- /dev/null +++ b/ingest_data_in_old_format/pipes/top_pages.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + Most visited pages for a given period. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calcualte hits and visits + +SQL > + + % + select + pathname, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + date >= toDate({{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}}) - interval 7 day + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + {% if defined(fixed_time) %} + and date <= {{Date(fixed_time, description="Way to set date and time. Useful parameter for testing.", required=False)}} + {% else %} + and date <= today() + {% end %} + {% end %} + group by + pathname + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} +TYPE endpoint + + diff --git a/ingest_data_in_old_format/pipes/top_sources.pipe b/ingest_data_in_old_format/pipes/top_sources.pipe new file mode 100644 index 00000000..0114cc08 --- /dev/null +++ b/ingest_data_in_old_format/pipes/top_sources.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Top traffic sources (domains), ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by referral and calcualte hits and visits + +SQL > + + % + select + domainWithoutWWW(referrer) as referrer, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + referrer + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + +TYPE endpoint diff --git a/ingest_data_in_old_format/pipes/trend.pipe b/ingest_data_in_old_format/pipes/trend.pipe new file mode 100644 index 00000000..4197bcbc --- /dev/null +++ b/ingest_data_in_old_format/pipes/trend.pipe @@ -0,0 +1,61 @@ +DESCRIPTION > + Visits trend over time for the last 30 minutes, filling the blanks. + Works great for the realtime chart. + + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the last 30 minutes, so we call fill empty data points + +SQL > + + with (now() - interval 30 minute) as start + select addMinutes(toStartOfMinute(start), number) as t + from ( + select arrayJoin(range(1, 31)) as number + ) + + + +NODE hits +DESCRIPTION > + Get last 30 minutes metrics gropued by minute + +SQL > + + select + toStartOfMinute(timestamp) as t, + uniq(session_id) as visits + from + analytics_hits + where + timestamp >= (now() - interval 30 minute) + group by + toStartOfMinute(timestamp) + order by + toStartOfMinute(timestamp) + + + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics for the last 30 minutes + +SQL > + + select + a.t, + b.visits + from + timeseries a + left join + hits b + on + a.t = b.t + order by + a.t + + +TYPE endpoint \ No newline at end of file diff --git a/ingest_data_in_old_format/requirements.txt b/ingest_data_in_old_format/requirements.txt new file mode 100644 index 00000000..421f0d2a --- /dev/null +++ b/ingest_data_in_old_format/requirements.txt @@ -0,0 +1 @@ +tinybird-cli==5.0.0.dev0 diff --git a/ingest_data_in_old_format/scripts/append_fixtures.sh b/ingest_data_in_old_format/scripts/append_fixtures.sh new file mode 100755 index 00000000..7ca463ab --- /dev/null +++ b/ingest_data_in_old_format/scripts/append_fixtures.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +directory="datasources/fixtures" +extensions=("csv" "ndjson") + +absolute_directory=$(realpath "$directory") + +for extension in "${extensions[@]}"; do + file_list=$(find "$absolute_directory" -type f -name "*.$extension") + + for file_path in $file_list; do + file_name=$(basename "$file_path") + file_name_without_extension="${file_name%.*}" + + command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name" + echo $command + $command + done +done diff --git a/ingest_data_in_old_format/scripts/exec_test.sh b/ingest_data_in_old_format/scripts/exec_test.sh new file mode 100755 index 00000000..8b018209 --- /dev/null +++ b/ingest_data_in_old_format/scripts/exec_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +export TB_VERSION_WARNING=0 + +run_test() { + t=$1 + echo "** Running $t **" + echo "** $(cat $t)" + if res=$(bash $t $2 | diff -B ${t}.result -); then + echo 'OK'; + else + echo "failed, diff:"; + echo "$res"; + return 1 + fi + echo "" +} +export -f run_test + +fail=0 +find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1 + +if [ $fail == 1 ]; then + exit -1; +fi diff --git a/ingest_data_in_old_format/tests/kpis.yaml b/ingest_data_in_old_format/tests/kpis.yaml new file mode 100644 index 00000000..ff20e4ef --- /dev/null +++ b/ingest_data_in_old_format/tests/kpis.yaml @@ -0,0 +1,41 @@ + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-12&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_single_day + description: Test single day with hourly granularity + parameters: date_from=2030-10-11&date_to=2030-10-13&fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_last_week + description: Test default behavior showing last 7 days + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + +- name: kpis_fixed_time + description: Test specific datetime with daily granularity + parameters: fixed_time=2030-10-15 + expected_result: | + {"date":"2030-10-08","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-09","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-11","visits":3,"pageviews":3,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-12","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2030-10-13","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-14","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} + {"date":"2030-10-15","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0} diff --git a/ingest_data_in_old_format/tests/top_browsers.yaml b/ingest_data_in_old_format/tests/top_browsers.yaml new file mode 100644 index 00000000..b6909ff4 --- /dev/null +++ b/ingest_data_in_old_format/tests/top_browsers.yaml @@ -0,0 +1,14 @@ + +- name: top_browsers_date_range + description: Test specific date range for browser statistics + parameters: date_from=2030-10-11&date_to=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} + +- name: top_browsers_default + description: Test top browsers with default parameters (last 7 days) + parameters: fixed_time=2030-10-13&skip=0&limit=20 + expected_result: | + {"browser":"chrome","visits":2,"hits":3} + {"browser":"firefox","visits":1,"hits":1} diff --git a/ingest_data_in_old_format/tests/top_pages.yaml b/ingest_data_in_old_format/tests/top_pages.yaml new file mode 100644 index 00000000..099b1523 --- /dev/null +++ b/ingest_data_in_old_format/tests/top_pages.yaml @@ -0,0 +1,14 @@ + +- name: top_pages_default + description: Test default values (last 7 days, limit 50, skip 0) + parameters: fixed_time=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1} + +- name: top_pages_date_range + description: Test specific date range + parameters: date_from=2030-10-11&date_to=2030-10-13 + expected_result: | + {"pathname":"\/product","visits":2,"hits":3} + {"pathname":"\/","visits":1,"hits":1}