From 55a238e83d9034c67d3ac51ad8288df4b3540481 Mon Sep 17 00:00:00 2001 From: David Holdeman Date: Sat, 6 Feb 2021 12:43:43 -0600 Subject: [PATCH] Decouple Doxygen, IBOMs, and Pinouts (#2293) * split workflow file * split scripts * add jobs * fix paths * fix job names * try * switch to push for testing * switch back to ftp, remove unneeded ppas * dont fetch submodules --- .github/workflows/gen-docs.yaml | 7 +- .github/workflows/gen-ibom.yaml | 37 ++++++++++ .github/workflows/gen-pinouts.yaml | 36 ++++++++++ .../generate_doxygen/gen_upload_docs.sh | 46 +------------ .../jenkins/generate_ibom/gen_upload_ibom.bat | 2 + misc/jenkins/generate_ibom/gen_upload_ibom.sh | 12 ++++ .../generate_pinouts/gen_upload_pinouts.bat | 2 + .../generate_pinouts/gen_upload_pinouts.sh | 27 ++++++++ misc/jenkins/setup/jobs/ibom/config.xml | 68 +++++++++++++++++++ misc/jenkins/setup/jobs/pinouts/config.xml | 68 +++++++++++++++++++ 10 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/gen-ibom.yaml create mode 100644 .github/workflows/gen-pinouts.yaml create mode 100644 misc/jenkins/generate_ibom/gen_upload_ibom.bat create mode 100644 misc/jenkins/generate_ibom/gen_upload_ibom.sh create mode 100644 misc/jenkins/generate_pinouts/gen_upload_pinouts.bat create mode 100644 misc/jenkins/generate_pinouts/gen_upload_pinouts.sh create mode 100644 misc/jenkins/setup/jobs/ibom/config.xml create mode 100644 misc/jenkins/setup/jobs/pinouts/config.xml diff --git a/.github/workflows/gen-docs.yaml b/.github/workflows/gen-docs.yaml index 271b858dce..47d280f59b 100644 --- a/.github/workflows/gen-docs.yaml +++ b/.github/workflows/gen-docs.yaml @@ -13,21 +13,16 @@ jobs: steps: - uses: actions/checkout@v2 with: - submodules: recursive fetch-depth: 0 - name: Install prerequisite software run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo add-apt-repository ppa:kicad/kicad-5.1-releases sudo apt-get update - sudo apt-get install ncftp kicad doxygen graphviz yq sshpass + sudo apt-get install doxygen graphviz ncftp - name: Set FTP variables env: ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - INTERACTIVE_HTML_BOM_NO_DISPLAY: 'true' run: | if [ "${{github.ref}}" = "refs/heads/master" ]; then echo "::set-env name=RUSEFI_FTP_SERVER::${{secrets.RUSEFI_FTP_SERVER}}"; diff --git a/.github/workflows/gen-ibom.yaml b/.github/workflows/gen-ibom.yaml new file mode 100644 index 0000000000..8efd371d53 --- /dev/null +++ b/.github/workflows/gen-ibom.yaml @@ -0,0 +1,37 @@ +name: Generate Interactive BOMs + +on: + push: + branches: master +# schedule: +# - cron: '0 0 * * *' + +jobs: + generate: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install prerequisite software + run: | + sudo add-apt-repository ppa:kicad/kicad-5.1-releases + sudo apt-get update + sudo apt-get install kicad ncftp + + - name: Set FTP variables + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' + run: | + if [ "${{github.ref}}" = "refs/heads/master" ]; then + echo "::set-env name=RUSEFI_FTP_SERVER::${{secrets.RUSEFI_FTP_SERVER}}"; + echo "::set-env name=RUSEFI_DOXYGEN_FTP_USER::${{secrets.RUSEFI_DOXYGEN_FTP_USER}}"; + echo "::set-env name=RUSEFI_DOXYGEN_FTP_PASS::${{secrets.RUSEFI_DOXYGEN_FTP_PASS}}"; + fi + + - name: Generate documentation + env: + INTERACTIVE_HTML_BOM_NO_DISPLAY: 'true' + run: bash ./misc/jenkins/generate_ibom/gen_upload_ibom.sh diff --git a/.github/workflows/gen-pinouts.yaml b/.github/workflows/gen-pinouts.yaml new file mode 100644 index 0000000000..a1b8cfc013 --- /dev/null +++ b/.github/workflows/gen-pinouts.yaml @@ -0,0 +1,36 @@ +name: Generate Interactive Pinouts + +on: + push: + branches: master +# schedule: +# - cron: '0 0 * * *' + +jobs: + generate: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install prerequisite software + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt-get update + sudo apt-get install yq ncftp + + - name: Set FTP variables + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' + run: | + if [ "${{github.ref}}" = "refs/heads/master" ]; then + echo "::set-env name=RUSEFI_FTP_SERVER::${{secrets.RUSEFI_FTP_SERVER}}"; + echo "::set-env name=RUSEFI_DOXYGEN_FTP_USER::${{secrets.RUSEFI_DOXYGEN_FTP_USER}}"; + echo "::set-env name=RUSEFI_DOXYGEN_FTP_PASS::${{secrets.RUSEFI_DOXYGEN_FTP_PASS}}"; + fi + + - name: Generate documentation + run: bash ./misc/jenkins/generate_pinouts/gen_upload_pinouts.sh diff --git a/misc/jenkins/generate_doxygen/gen_upload_docs.sh b/misc/jenkins/generate_doxygen/gen_upload_docs.sh index 6fd322786c..f8e55c1c36 100644 --- a/misc/jenkins/generate_doxygen/gen_upload_docs.sh +++ b/misc/jenkins/generate_doxygen/gen_upload_docs.sh @@ -1,17 +1,5 @@ #!/bin/bash -echo "Should be executed from project root folder. Will try to upload to $RUSEFI_FTP_SERVER" -pwd -# ibom is part of Doxygen job simply in order to reduce workspace HDD usage on my tiny build server -bash misc/jenkins/InteractiveHtmlBom/run.sh - -if [ -n "$RUSEFI_FTP_SERVER" ]; then - echo "Uploading IBOMs" - tar -cvzf - -C hardware ibom |\ - sshpass -p $RUSEFI_DOXYGEN_FTP_PASS ssh -o StrictHostKeyChecking=no $RUSEFI_DOXYGEN_FTP_USER@$RUSEFI_FTP_SERVER \ - "(tar -xzvf - -C ~)" -fi - pwd cd firmware @@ -22,38 +10,6 @@ doxygen || { echo "doxygen run FAILED"; exit 1; } cd ../doxygen if [ -n "$RUSEFI_FTP_SERVER" ]; then echo "Uploading Doxygen" - tar -cvzf - html |\ - sshpass -p $RUSEFI_DOXYGEN_FTP_PASS ssh -o StrictHostKeyChecking=no $RUSEFI_DOXYGEN_FTP_USER@$RUSEFI_FTP_SERVER \ - "(tar -xzvf - -C ~)" -fi -[ $? -eq 0 ] || { echo "upload FAILED"; exit 1; } - -cd .. - -CONNECTORS=$(find -path "./firmware/config/boards/*/connectors/*.yaml") -for c in $CONNECTORS; do - echo "processing "$c - DIR="pinouts/"$(echo $c | tr '/' '\n' | tail -n +5 | head -n -2 | tr '\n' '/') - echo "DIR "$DIR - NAME=$(basename $c .yaml) - echo "NAME "$NAME - mkdir -p $DIR - bash misc/pinout-gen/gen.sh $c > $DIR/$NAME.html - file $DIR/$NAME.html - IMG=$(yq r $c 'info.image.file') - echo "IMG "$IMG - if [ $IMG ]; then - cp $(dirname $c)/$IMG $DIR - fi - ls $DIR - cp misc/pinout-gen/yaml.min.js $DIR - ls $DIR -done - -if [ -n "$RUSEFI_FTP_SERVER" ]; then - echo "Uploading Pinouts" - tar -cvzf - pinouts |\ - sshpass -p $RUSEFI_DOXYGEN_FTP_PASS ssh -o StrictHostKeyChecking=no $RUSEFI_DOXYGEN_FTP_USER@$RUSEFI_FTP_SERVER \ - "(tar -xzvf - -C ~)" + ncftpput -R -m -u "$RUSEFI_DOXYGEN_FTP_USER" -p "$RUSEFI_DOXYGEN_FTP_PASS" "$RUSEFI_FTP_SERVER" /html html/* fi [ $? -eq 0 ] || { echo "upload FAILED"; exit 1; } diff --git a/misc/jenkins/generate_ibom/gen_upload_ibom.bat b/misc/jenkins/generate_ibom/gen_upload_ibom.bat new file mode 100644 index 0000000000..fccd397af0 --- /dev/null +++ b/misc/jenkins/generate_ibom/gen_upload_ibom.bat @@ -0,0 +1,2 @@ +@echo off +bash.exe misc\jenkins\generate_ibom\gen_upload_ibom.sh diff --git a/misc/jenkins/generate_ibom/gen_upload_ibom.sh b/misc/jenkins/generate_ibom/gen_upload_ibom.sh new file mode 100644 index 0000000000..df42b73d12 --- /dev/null +++ b/misc/jenkins/generate_ibom/gen_upload_ibom.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "Should be executed from project root folder. Will try to upload to $RUSEFI_FTP_SERVER" +pwd +# ibom is part of Doxygen job simply in order to reduce workspace HDD usage on my tiny build server +bash misc/jenkins/InteractiveHtmlBom/run.sh + +if [ -n "$RUSEFI_FTP_SERVER" ]; then + echo "Uploading IBOMs" + ncftpput -R -m -u "$RUSEFI_DOXYGEN_FTP_USER" -p "$RUSEFI_DOXYGEN_FTP_PASS" "$RUSEFI_FTP_SERVER" /ibom hardware/ibom/* +fi +[ $? -eq 0 ] || { echo "upload FAILED"; exit 1; } diff --git a/misc/jenkins/generate_pinouts/gen_upload_pinouts.bat b/misc/jenkins/generate_pinouts/gen_upload_pinouts.bat new file mode 100644 index 0000000000..dbdd6f4769 --- /dev/null +++ b/misc/jenkins/generate_pinouts/gen_upload_pinouts.bat @@ -0,0 +1,2 @@ +@echo off +bash.exe misc\jenkins\generate_pinouts\gen_upload_pinouts.sh diff --git a/misc/jenkins/generate_pinouts/gen_upload_pinouts.sh b/misc/jenkins/generate_pinouts/gen_upload_pinouts.sh new file mode 100644 index 0000000000..8802419b67 --- /dev/null +++ b/misc/jenkins/generate_pinouts/gen_upload_pinouts.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +CONNECTORS=$(find -path "./firmware/config/boards/*/connectors/*.yaml") +for c in $CONNECTORS; do + echo "processing "$c + DIR="pinouts/"$(echo $c | tr '/' '\n' | tail -n +5 | head -n -2 | tr '\n' '/') + echo "DIR "$DIR + NAME=$(basename $c .yaml) + echo "NAME "$NAME + mkdir -p $DIR + bash misc/pinout-gen/gen.sh $c > $DIR/$NAME.html + file $DIR/$NAME.html + IMG=$(yq r $c 'info.image.file') + echo "IMG "$IMG + if [ $IMG ]; then + cp $(dirname $c)/$IMG $DIR + fi + ls $DIR + cp misc/pinout-gen/yaml.min.js $DIR + ls $DIR +done + +if [ -n "$RUSEFI_FTP_SERVER" ]; then + echo "Uploading Pinouts" + ncftpput -R -m -u "$RUSEFI_DOXYGEN_FTP_USER" -p "$RUSEFI_DOXYGEN_FTP_PASS" "$RUSEFI_FTP_SERVER" /pinouts pinouts/* +fi +[ $? -eq 0 ] || { echo "upload FAILED"; exit 1; } diff --git a/misc/jenkins/setup/jobs/ibom/config.xml b/misc/jenkins/setup/jobs/ibom/config.xml new file mode 100644 index 0000000000..ee6ca71f03 --- /dev/null +++ b/misc/jenkins/setup/jobs/ibom/config.xml @@ -0,0 +1,68 @@ + + + + + false + + + + -1 + 100 + -1 + -1 + + + + + 2 + + + https://github.com/rusefi/rusefi + + + + + */master + + + false + + + + true + false + false + false + + + 10 03 * * * + false + + + false + + + call misc\jenkins\generate_ibom\gen_upload_ibom.bat + + + + + 30 + -1 + + -1 + -1 + -1 + -1 + + + + + + false + + + false + + + diff --git a/misc/jenkins/setup/jobs/pinouts/config.xml b/misc/jenkins/setup/jobs/pinouts/config.xml new file mode 100644 index 0000000000..e6cd4e676d --- /dev/null +++ b/misc/jenkins/setup/jobs/pinouts/config.xml @@ -0,0 +1,68 @@ + + + + + false + + + + -1 + 100 + -1 + -1 + + + + + 2 + + + https://github.com/rusefi/rusefi + + + + + */master + + + false + + + + true + false + false + false + + + 10 03 * * * + false + + + false + + + call misc\jenkins\generate_pinouts\gen_upload_pinouts.bat + + + + + 30 + -1 + + -1 + -1 + -1 + -1 + + + + + + false + + + false + + +