From 4ac57b3db820fbcb1838d668a68cd502cc8bc368 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Tue, 11 Jun 2024 15:27:52 -0700 Subject: [PATCH 01/10] When uncompressing a tgz into a package, delete the tgz after uncompressing it. --- .github/workflows/cpp-packaging.yml | 2 +- build_scripts/desktop/package.sh | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 041ef50bb2..0e78c7cb92 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -567,7 +567,7 @@ jobs: # Windows x86: force input and output target format additional_flags+=(-f pe-i386,pe-bigobj-i386) fi - sdk-src/build_scripts/desktop/package.sh -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]} + sdk-src/build_scripts/desktop/package.sh -D -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]} done if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then # Darwin has a final step after all the variants are done, diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index cef0a16787..b571d7db4f 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -21,6 +21,7 @@ options: -L, use LLVM binutils -R, print rename prefix and exit -N, print allowed namespaces and exit + -D, delete package files after using them to save disk space example: build_scripts/desktop/package.sh -b firebase-cpp-sdk-linux -p linux -o package_out -v x86 -j" } @@ -31,6 +32,7 @@ platform= python_cmd=python variant=. verbose=0 +delete_files_after_using=0 root_dir=$(cd $(dirname $0)/../..; pwd -P) merge_libraries_script=${root_dir}/scripts/merge_libraries.py tools_path=~/bin @@ -57,7 +59,7 @@ abspath(){ fi } -while getopts "f:b:o:p:d:m:P:t:NRhjLv" opt; do +while getopts "f:b:o:p:d:m:P:t:NRhjLDv" opt; do case $opt in f) binutils_format=$OPTARG @@ -65,6 +67,9 @@ while getopts "f:b:o:p:d:m:P:t:NRhjLv" opt; do b) built_sdk_path=$OPTARG ;; + D) + delete_files_after_using=1 + ;; j) run_in_parallel=1 ;; @@ -158,6 +163,9 @@ if [[ ! -d "${built_sdk_path}" && -f "${built_sdk_path}" ]]; then echo "Uncompressing tarfile into temporary directory..." tar -xf "${built_sdk_path}" -C "${temp_dir}" built_sdk_path="${temp_dir}" + if [[ ${delete_files_after_using} -eq 1 ]]; then + rm -f "${built_sdk_path}" + fi fi if [[ ! -r "${built_sdk_path}/CMakeCache.txt" ]]; then @@ -363,13 +371,13 @@ for product in ${product_list[*]}; do --scan_libs=\"${allfiles}\" \\ --hide_c_symbols=\"${deps_hidden}\" \\ \"${libfile_src}\" ${deps[*]}" >> "${merge_libraries_tmp}/merge_${product}.sh" - chmod u+x "${merge_libraries_tmp}/merge_${product}.sh" - if [[ ${run_in_parallel} -eq 0 ]]; then - # Run immediately if not set to run in parallel. - "${merge_libraries_tmp}/merge_${product}.sh" - else - echo "echo \"${libfile_out}\" DONE" >> "${merge_libraries_tmp}/merge_${product}.sh" - fi + chmod u+x "${merge_libraries_tmp}/merge_${product}.sh" + if [[ ${run_in_parallel} -eq 0 ]]; then + # Run immediately if not set to run in parallel. + "${merge_libraries_tmp}/merge_${product}.sh" + else + echo "echo \"${libfile_out}\" DONE" >> "${merge_libraries_tmp}/merge_${product}.sh" + fi done if [[ ${run_in_parallel} -ne 0 ]]; then From 1ae4659d312145cd97a54d80d73a0bbb7d7c6b32 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Tue, 11 Jun 2024 16:47:28 -0700 Subject: [PATCH 02/10] Fix bug. --- build_scripts/desktop/package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index b571d7db4f..7c809052d1 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -162,10 +162,10 @@ if [[ ! -d "${built_sdk_path}" && -f "${built_sdk_path}" ]]; then trap "rm -rf \"\${temp_dir}\"" SIGKILL SIGTERM SIGQUIT EXIT echo "Uncompressing tarfile into temporary directory..." tar -xf "${built_sdk_path}" -C "${temp_dir}" - built_sdk_path="${temp_dir}" if [[ ${delete_files_after_using} -eq 1 ]]; then rm -f "${built_sdk_path}" fi + built_sdk_path="${temp_dir}" fi if [[ ! -r "${built_sdk_path}/CMakeCache.txt" ]]; then From 14e05b33722cfe6a217ad9b1d6a726b9a2ec9980 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Wed, 12 Jun 2024 11:07:31 -0700 Subject: [PATCH 03/10] Add some disk space printing. --- .github/workflows/cpp-packaging.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 0e78c7cb92..b484a06d17 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -541,6 +541,7 @@ jobs: if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then verbose_flag=-v fi + du -skch || true declare -a additional_flags tar -xvzf artifacts/packaging-tools-${tools_platform}/packaging-tools.tgz -C bin chmod -R u+x bin @@ -567,6 +568,7 @@ jobs: # Windows x86: force input and output target format additional_flags+=(-f pe-i386,pe-bigobj-i386) fi + du -skch || true sdk-src/build_scripts/desktop/package.sh -D -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]} done if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then From 4b26fbafdbd6fddcf9f3b8ebae03e78831c6169f Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Wed, 12 Jun 2024 11:07:55 -0700 Subject: [PATCH 04/10] Format. --- .github/workflows/cpp-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index b484a06d17..5a902cdba4 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -541,7 +541,7 @@ jobs: if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then verbose_flag=-v fi - du -skch || true + du -skch || true declare -a additional_flags tar -xvzf artifacts/packaging-tools-${tools_platform}/packaging-tools.tgz -C bin chmod -R u+x bin From 5a31e898b13868b6ad3a698590662ed689956974 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Wed, 12 Jun 2024 11:57:51 -0700 Subject: [PATCH 05/10] Remove libraries after they are blastdoored. Add more debug printing of disk space usage. --- .github/workflows/cpp-packaging.yml | 5 +++++ build_scripts/desktop/package.sh | 3 +++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 5a902cdba4..ab27e3c280 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -542,8 +542,10 @@ jobs: verbose_flag=-v fi du -skch || true + df -h || true declare -a additional_flags tar -xvzf artifacts/packaging-tools-${tools_platform}/packaging-tools.tgz -C bin + rm -rf artifacts/packaging-tools-*.tgz chmod -R u+x bin # To save space, delete any artifacts that we don't need for packaging. for pkg in artifacts/firebase-cpp-sdk-*; do @@ -552,6 +554,8 @@ jobs: rm -rf "${pkg}" fi done + du -skch || true + df -h || true for pkg in artifacts/firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}*-build/*.tgz; do # determine the build variant based on the artifact filename variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}") @@ -569,6 +573,7 @@ jobs: additional_flags+=(-f pe-i386,pe-bigobj-i386) fi du -skch || true + df -h || true sdk-src/build_scripts/desktop/package.sh -D -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]} done if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index 7c809052d1..a741931e6e 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -371,6 +371,9 @@ for product in ${product_list[*]}; do --scan_libs=\"${allfiles}\" \\ --hide_c_symbols=\"${deps_hidden}\" \\ \"${libfile_src}\" ${deps[*]}" >> "${merge_libraries_tmp}/merge_${product}.sh" + if [[ ${delete_files_after_using} -eq 1 ]]; then + echo "rm -f ${deps[*]}" >> "${merge_libraries_tmp}/merge_${product}.sh" + fi chmod u+x "${merge_libraries_tmp}/merge_${product}.sh" if [[ ${run_in_parallel} -eq 0 ]]; then # Run immediately if not set to run in parallel. From 698948bff3943d769d42ac3144ff19651d3cd433 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Wed, 12 Jun 2024 13:27:49 -0700 Subject: [PATCH 06/10] Comment out --build_tests on windows to see what happens. --- .github/workflows/cpp-packaging.yml | 4 ++-- build_scripts/desktop/package.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index ab27e3c280..4ea2b33da9 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -314,12 +314,12 @@ jobs: include: - os: windows-latest vcpkg_triplet_suffix: "windows-static" - additional_build_flags: "--build_tests" + #additional_build_flags: "--build_tests" sdk_platform: "windows" - os: windows-latest msvc_runtime: "dynamic" vcpkg_triplet_suffix: "windows-static-md" - additional_build_flags: "--build_tests" + #additional_build_flags: "--build_tests" sdk_platform: "windows" - os: ubuntu-20.04 vcpkg_triplet_suffix: "linux" diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index a741931e6e..6a33bc8fd8 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -204,6 +204,11 @@ else ext='a' fi +if [[ ${delete_files_after_using} -eq 1 ]]; then + # Delete all non-library, non-header files. + find "${built_sdk_path}" -not -name "*.${ext}" -and -not -name "*.h" -print0 | xargs -0 rm +fi + # Library dependencies to merge. Each should be a whitespace-delimited list of path globs. readonly deps_firebase_app=" */${prefix}firebase_rest_lib.${ext} From fc675be0c684003be765c7d4fc381eeff379e8dd Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Wed, 12 Jun 2024 14:23:11 -0700 Subject: [PATCH 07/10] Don't delete non-library files. --- build_scripts/desktop/package.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index 6a33bc8fd8..65acd38737 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -204,10 +204,10 @@ else ext='a' fi -if [[ ${delete_files_after_using} -eq 1 ]]; then - # Delete all non-library, non-header files. - find "${built_sdk_path}" -not -name "*.${ext}" -and -not -name "*.h" -print0 | xargs -0 rm -fi +#if [[ ${delete_files_after_using} -eq 1 ]]; then +# # Delete all non-library, non-header files. +# find "${built_sdk_path}" -type f -not -name "*.${ext}" -and -not -name "*.h" -print0 | xargs -0 rm +#fi # Library dependencies to merge. Each should be a whitespace-delimited list of path globs. readonly deps_firebase_app=" From d2ce8cad5b02b12e388d7c4a734a128447950cab Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Thu, 13 Jun 2024 11:20:54 -0700 Subject: [PATCH 08/10] Remove disk space printing. --- .github/workflows/cpp-packaging.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 4ea2b33da9..32f9a89a40 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -541,8 +541,6 @@ jobs: if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then verbose_flag=-v fi - du -skch || true - df -h || true declare -a additional_flags tar -xvzf artifacts/packaging-tools-${tools_platform}/packaging-tools.tgz -C bin rm -rf artifacts/packaging-tools-*.tgz @@ -554,8 +552,6 @@ jobs: rm -rf "${pkg}" fi done - du -skch || true - df -h || true for pkg in artifacts/firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}*-build/*.tgz; do # determine the build variant based on the artifact filename variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}") @@ -572,8 +568,6 @@ jobs: # Windows x86: force input and output target format additional_flags+=(-f pe-i386,pe-bigobj-i386) fi - du -skch || true - df -h || true sdk-src/build_scripts/desktop/package.sh -D -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]} done if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then From 161d252d208c7559b4dd9911b1ff6ef9f8a293ad Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Thu, 13 Jun 2024 11:22:21 -0700 Subject: [PATCH 09/10] Remove commented-out code. --- build_scripts/desktop/package.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build_scripts/desktop/package.sh b/build_scripts/desktop/package.sh index 65acd38737..a741931e6e 100755 --- a/build_scripts/desktop/package.sh +++ b/build_scripts/desktop/package.sh @@ -204,11 +204,6 @@ else ext='a' fi -#if [[ ${delete_files_after_using} -eq 1 ]]; then -# # Delete all non-library, non-header files. -# find "${built_sdk_path}" -type f -not -name "*.${ext}" -and -not -name "*.h" -print0 | xargs -0 rm -#fi - # Library dependencies to merge. Each should be a whitespace-delimited list of path globs. readonly deps_firebase_app=" */${prefix}firebase_rest_lib.${ext} From 443f1aa009f9383d1d1bdcb2169f7d5f365dfbe0 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Thu, 13 Jun 2024 11:24:38 -0700 Subject: [PATCH 10/10] Removed commented lines. --- .github/workflows/cpp-packaging.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 32f9a89a40..a19864449e 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -314,12 +314,10 @@ jobs: include: - os: windows-latest vcpkg_triplet_suffix: "windows-static" - #additional_build_flags: "--build_tests" sdk_platform: "windows" - os: windows-latest msvc_runtime: "dynamic" vcpkg_triplet_suffix: "windows-static-md" - #additional_build_flags: "--build_tests" sdk_platform: "windows" - os: ubuntu-20.04 vcpkg_triplet_suffix: "linux"