Add nightly dependency bump workflow

.github/workflows/main-build.yml

@@ -6,7 +6,14 @@ on:
       - main
       - "release/v*"
       - ci-workflow
+      - nightly-dependency-updates
   workflow_dispatch: # be able to run the workflow on demand
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref to checkout'
+        required: false
+        type: string
 env:
   AWS_DEFAULT_REGION: us-east-1
   STAGING_ECR_REGISTRY: 637423224110.dkr.ecr.us-east-1.amazonaws.com
@@ -34,6 +41,8 @@ jobs:
     steps:
       - name: Checkout Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0
+        with:
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Get Python Distro Output
         id: python_output
@@ -114,7 +123,7 @@ jobs:
     name: "Publish Main Build Status"
     needs: [ build, application-signals-e2e-test ]
     runs-on: ubuntu-latest
-    if: always()
+    if: always() && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/'))
     steps:
       - name: Configure AWS Credentials for emitting metrics
         uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0

.github/workflows/nightly-build.yml

@@ -0,0 +1,97 @@
+name: Nightly Upstream Snapshot Build
+
+on:
+  schedule:
+    - cron: "21 3 * * *"
+  workflow_dispatch:
+
+env:
+  BRANCH_NAME: nightly-dependency-updates
+
+jobs:
+  update-and-create-pr:
+    runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.check_changes.outputs.has_changes }}
+      otel_python_version: ${{ steps.get_versions.outputs.otel_python_version }}
+      otel_contrib_version: ${{ steps.get_versions.outputs.otel_contrib_version }}
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0
+      with:
+        fetch-depth: 0
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Check if nightly branch already exists
+      run: |
+        if git ls-remote --exit-code --heads origin "$BRANCH_NAME"; then
+          echo "Branch $BRANCH_NAME already exists. Skipping run to avoid conflicts."
+          echo "Please merge or close the existing PR before the next nightly run."
+          exit 1
+        fi
+    
+    - name: Configure git and create branch
+      run: |
+        git config --local user.email "action@github.com"
+        git config --local user.name "GitHub Action"
+        git checkout -b "$BRANCH_NAME"
+    
+    - name: Set up Python
+      uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0
+      with:
+        python-version: '3.11'
+
+    - name: Install build tools
+      run: |
+        python -m pip install --upgrade pip
+        pip install toml requests
+    
+    - name: Get latest upstream versions
+      id: get_versions
+      run: python scripts/get_upstream_versions.py
+
+    - name: Update dependencies
+      env:
+        OTEL_PYTHON_VERSION: ${{ steps.get_versions.outputs.otel_python_version }}
+        OTEL_CONTRIB_VERSION: ${{ steps.get_versions.outputs.otel_contrib_version }}
+      run: python scripts/update_dependencies.py
+
+    - name: Check for changes and create PR
+      id: check_changes
+      run: |
+        if git diff --quiet; then
+          echo "No dependency updates needed"
+          echo "has_changes=false" >> $GITHUB_OUTPUT
+        else
+          echo "Dependencies were updated"
+          echo "has_changes=true" >> $GITHUB_OUTPUT
+          
+          # Create PR
+          git add aws-opentelemetry-distro/pyproject.toml
+          git commit -m "chore: update OpenTelemetry dependencies to ${{ steps.get_versions.outputs.otel_python_version }}/${{ steps.get_versions.outputs.otel_contrib_version }}"
+          git push origin "$BRANCH_NAME"
+          
+          gh pr create \
+            --title "Nightly dependency update: OpenTelemetry ${{ steps.get_versions.outputs.otel_python_version }}/${{ steps.get_versions.outputs.otel_contrib_version }}" \
+            --body "Automated update of OpenTelemetry dependencies.
+
+          **Updated versions:**
+          - OpenTelemetry Python: ${{ steps.get_versions.outputs.otel_python_version }}
+          - OpenTelemetry Contrib: ${{ steps.get_versions.outputs.otel_contrib_version }}" \
+            --base main \
+            --head "$BRANCH_NAME"
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  build-and-test:
+    needs: update-and-create-pr
+    if: needs.update-and-create-pr.outputs.has_changes == 'true'
+    uses: ./.github/workflows/main-build.yml
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      ref: nightly-dependency-updates

aws-opentelemetry-distro/pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
 dependencies = [
   "opentelemetry-api == 1.33.1",
   "opentelemetry-sdk == 1.33.1",
+  "opentelemetry-exporter-otlp == 1.33.1",
   "opentelemetry-exporter-otlp-proto-grpc == 1.33.1",
   "opentelemetry-exporter-otlp-proto-http == 1.33.1",
   "opentelemetry-propagator-b3 == 1.33.1",

scripts/get_upstream_versions.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import sys
+
+import requests
+
+
+def get_latest_otel_versions():
+    """Get latest OpenTelemetry versions from GitHub releases."""
+    try:
+        # Query GitHub API for latest release
+        response = requests.get(
+            "https://api.github.com/repos/open-telemetry/opentelemetry-python/releases/latest", timeout=30
+        )
+        response.raise_for_status()
+
+        release_data = response.json()
+        release_title = release_data["name"]
+
+        # Parse "Version 1.37.0/0.58b0" format
+        match = re.search(r"Version\s+(\d+\.\d+\.\d+)/(\d+\.\d+b\d+)", release_title)
+        if not match:
+            print(f"Could not parse release title: {release_title}")
+            sys.exit(1)
+
+        otel_python_version = match.group(1)
+        otel_contrib_version = match.group(2)
+
+        return otel_python_version, otel_contrib_version
+
+    except requests.RequestException as request_error:
+        print(f"Error getting OpenTelemetry versions: {request_error}")
+        sys.exit(1)
+
+
+def main():
+    otel_python_version, otel_contrib_version = get_latest_otel_versions()
+
+    print(f"OTEL_PYTHON_VERSION={otel_python_version}")
+    print(f"OTEL_CONTRIB_VERSION={otel_contrib_version}")
+
+    # Write to GitHub output if in CI
+    if "GITHUB_OUTPUT" in os.environ:
+        with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file:
+            output_file.write(f"otel_python_version={otel_python_version}\n")
+            output_file.write(f"otel_contrib_version={otel_contrib_version}\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/update_dependencies.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import sys
+
+import requests
+
+# Dependencies that use the first version number (opentelemetry-python)
+PYTHON_CORE_DEPS = [
+    "opentelemetry-api",
+    "opentelemetry-sdk",
+    "opentelemetry-exporter-otlp",
+    "opentelemetry-exporter-otlp-proto-grpc",
+    "opentelemetry-exporter-otlp-proto-http",
+    "opentelemetry-propagator-b3",
+    "opentelemetry-propagator-jaeger",
+    "opentelemetry-exporter-otlp-proto-common",
+]
+
+# Dependencies that use the second version number (opentelemetry-python-contrib)
+CONTRIB_DEPS = [
+    "opentelemetry-distro",
+    "opentelemetry-processor-baggage",
+    "opentelemetry-propagator-ot-trace",
+    "opentelemetry-instrumentation",
+    "opentelemetry-instrumentation-aws-lambda",
+    "opentelemetry-instrumentation-aio-pika",
+    "opentelemetry-instrumentation-aiohttp-client",
+    "opentelemetry-instrumentation-aiopg",
+    "opentelemetry-instrumentation-asgi",
+    "opentelemetry-instrumentation-asyncpg",
+    "opentelemetry-instrumentation-boto",
+    "opentelemetry-instrumentation-boto3sqs",
+    "opentelemetry-instrumentation-botocore",
+    "opentelemetry-instrumentation-celery",
+    "opentelemetry-instrumentation-confluent-kafka",
+    "opentelemetry-instrumentation-dbapi",
+    "opentelemetry-instrumentation-django",
+    "opentelemetry-instrumentation-elasticsearch",
+    "opentelemetry-instrumentation-falcon",
+    "opentelemetry-instrumentation-fastapi",
+    "opentelemetry-instrumentation-flask",
+    "opentelemetry-instrumentation-grpc",
+    "opentelemetry-instrumentation-httpx",
+    "opentelemetry-instrumentation-jinja2",
+    "opentelemetry-instrumentation-kafka-python",
+    "opentelemetry-instrumentation-logging",
+    "opentelemetry-instrumentation-mysql",
+    "opentelemetry-instrumentation-mysqlclient",
+    "opentelemetry-instrumentation-pika",
+    "opentelemetry-instrumentation-psycopg2",
+    "opentelemetry-instrumentation-pymemcache",
+    "opentelemetry-instrumentation-pymongo",
+    "opentelemetry-instrumentation-pymysql",
+    "opentelemetry-instrumentation-pyramid",
+    "opentelemetry-instrumentation-redis",
+    "opentelemetry-instrumentation-remoulade",
+    "opentelemetry-instrumentation-requests",
+    "opentelemetry-instrumentation-sqlalchemy",
+    "opentelemetry-instrumentation-sqlite3",
+    "opentelemetry-instrumentation-starlette",
+    "opentelemetry-instrumentation-system-metrics",
+    "opentelemetry-instrumentation-tornado",
+    "opentelemetry-instrumentation-tortoiseorm",
+    "opentelemetry-instrumentation-urllib",
+    "opentelemetry-instrumentation-urllib3",
+    "opentelemetry-instrumentation-wsgi",
+    "opentelemetry-instrumentation-cassandra",
+]
+
+# AWS-specific packages with independent versioning
+AWS_DEPS = [
+    "opentelemetry-sdk-extension-aws",
+    "opentelemetry-propagator-aws-xray",
+]
+
+
+def get_latest_version(package_name):
+    """Get the latest version of a package from PyPI."""
+    try:
+        response = requests.get(f"https://pypi.org/pypi/{package_name}/json", timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        return data["info"]["version"]
+    except requests.RequestException as request_error:
+        print(f"Warning: Could not get latest version for {package_name}: {request_error}")
+        return None
+
+
+def main():
+    otel_python_version = os.environ.get("OTEL_PYTHON_VERSION")
+    otel_contrib_version = os.environ.get("OTEL_CONTRIB_VERSION")
+
+    if not otel_python_version or not otel_contrib_version:
+        print("Error: OTEL_PYTHON_VERSION and OTEL_CONTRIB_VERSION environment variables required")
+        sys.exit(1)
+
+    pyproject_path = "aws-opentelemetry-distro/pyproject.toml"
+
+    try:
+        with open(pyproject_path, "r", encoding="utf-8") as input_file:
+            content = input_file.read()
+
+        updated = False
+
+        # Update opentelemetry-python dependencies
+        for dep in PYTHON_CORE_DEPS:
+            pattern = rf'"{re.escape(dep)} == [^"]*"'
+            replacement = f'"{dep} == {otel_python_version}"'
+            if re.search(pattern, content):
+                content = re.sub(pattern, replacement, content)
+                updated = True
+
+        # Update opentelemetry-python-contrib dependencies
+        for dep in CONTRIB_DEPS:
+            pattern = rf'"{re.escape(dep)} == [^"]*"'
+            replacement = f'"{dep} == {otel_contrib_version}"'
+            if re.search(pattern, content):
+                content = re.sub(pattern, replacement, content)
+                updated = True
+
+        # Update dependencies with independent versioning
+        for dep in AWS_DEPS:
+            latest_version = get_latest_version(dep)
+            if latest_version:
+                pattern = rf'"{re.escape(dep)} == [^"]*"'
+                replacement = f'"{dep} == {latest_version}"'
+                if re.search(pattern, content):
+                    content = re.sub(pattern, replacement, content)
+                    updated = True
+                    print(f"Updated {dep} to {latest_version}")
+
+        if updated:
+            with open(pyproject_path, "w", encoding="utf-8") as output_file:
+                output_file.write(content)
+            print(f"Dependencies updated to Python {otel_python_version} / Contrib {otel_contrib_version}")
+        else:
+            print("No OpenTelemetry dependencies found to update")
+
+    except (OSError, IOError) as file_error:
+        print(f"Error updating dependencies: {file_error}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()