diff --git a/builds/checkin/e2e-checkin.yaml b/builds/checkin/e2e-checkin.yaml new file mode 100644 index 00000000000..f4786d33eef --- /dev/null +++ b/builds/checkin/e2e-checkin.yaml @@ -0,0 +1,59 @@ +trigger: none +pr: + branches: + include: + - master + - release/* + +resources: + pipelines: + - pipeline: images + source: 'Azure-IoT-Edge-Core Build Images' + branch: 'release/1.2' + - pipeline: packages + source: 'Azure-IoT-Edge-Core Edgelet Packages' + branch: 'release/1.2' + +stages: + + - template: ../misc/templates/build-images.yaml + parameters: + E2EBuild : true + + - template: ../misc/templates/build-packages.yaml + parameters: + E2EBuild : true + + - stage: RunE2ETest + displayName: End-to-End Test + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-20.04-msmoby + dependsOn: + - PublishManifests + - BuildPackages + condition: | + and + ( + succeededOrFailed(), + in(dependencies.PublishManifests.result, 'Succeeded','Skipped'), + in(dependencies.BuildPackages.result, 'Succeeded','Skipped') + ) + jobs: + - job: ubuntu_2004_msmoby + displayName: Ubuntu 20.04 with iotedge-moby + variables: + verbose: false + os: linux + arch: amd64 + artifactName: iotedged-ubuntu20.04-amd64 + identityServiceArtifactName: packages_ubuntu-20.04_amd64 + identityServicePackageFilter: aziot-identity-service_*_amd64.deb + builtImages: $[ stageDependencies.PublishManifests.PublishManifest.result ] + builtPackages: $[ stageDependencies.BuildPackages.linux.result ] + steps: + - template: ../e2e/templates/e2e-setup.yaml + - template: ../e2e/templates/e2e-run.yaml + + \ No newline at end of file diff --git a/builds/e2e/templates/e2e-setup.yaml b/builds/e2e/templates/e2e-setup.yaml index 65a158eb65b..55dfe5d2c55 100644 --- a/builds/e2e/templates/e2e-setup.yaml +++ b/builds/e2e/templates/e2e-setup.yaml @@ -1,3 +1,4 @@ + steps: - checkout: self clean: true @@ -22,9 +23,48 @@ steps: TestRootCaPassword, TestBlobStoreSas -- task: DownloadBuildArtifacts@0 - displayName: Get Docker image info +- bash: | + echo "Built Images Result is $(builtImages)" + echo "Built Packages Result is $(builtPackages)" + if [[ ! -z $(builtImages) && $(builtImages) == 'Succeeded' ]] + then + echo "Use Pipeline Image Artifacts" + echo "##vso[task.setvariable variable=UsePipelineImageArtifacts;isOutput=true]TRUE" + else + echo "Use Build Image Artifacts" + echo "##vso[task.setvariable variable=UsePipelineImageArtifacts;isOutput=true]FALSE" + fi + if [[ ! -z $(builtPackages) && $(builtPackages) == 'Succeeded' ]] + then + echo "Use Pipeline Package Artifacts" + echo "##vso[task.setvariable variable=UsePipelinePackageArtifacts;isOutput=true]TRUE" + else + echo "Use Build Package Artifacts" + echo "##vso[task.setvariable variable=UsePipelinePackageArtifacts;isOutput=true]FALSE" + fi + name: check_use_artifacts + displayName: Check Use Pipeline Image Artifacts + +- task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Build Images + condition: eq(variables['check_use_artifacts.UsePipelineImageArtifacts'],'true') + inputs: + source: 'current' + path: $(System.ArtifactsDirectory) + patterns: $(az.pipeline.images.artifacts)/artifactInfo.txt + +- task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Build Packages + condition: eq(variables['check_use_artifacts.UsePipelinePackageArtifacts'],'true') inputs: + source: 'current' + path: $(System.ArtifactsDirectory) + patterns: $(artifactName)/*.deb + +- task: DownloadBuildArtifacts@0 + displayName: Download Resource Build Images + condition: eq(variables['check_use_artifacts.UsePipelineImageArtifacts'],'false') + inputs: buildType: specific project: $(resources.pipeline.images.projectID) pipeline: $(resources.pipeline.images.pipelineName) @@ -34,9 +74,10 @@ steps: artifactName: $(az.pipeline.images.artifacts) allowPartiallySucceededBuilds: true itemPattern: $(az.pipeline.images.artifacts)/artifactInfo.txt - + - task: DownloadBuildArtifacts@0 displayName: Download edgelet packages + condition: eq(variables['check_use_artifacts.UsePipelinePackageArtifacts'],'false') inputs: buildType: specific project: $(resources.pipeline.packages.projectID) diff --git a/builds/misc/images.yaml b/builds/misc/images.yaml index 6a0b7e42dfe..81137c0c54f 100644 --- a/builds/misc/images.yaml +++ b/builds/misc/images.yaml @@ -7,424 +7,4 @@ trigger: pr: none stages: -################################################################################ -- stage: BuildExecutables -################################################################################ - displayName: Build Excecutables - pool: - name: $(pool.linux.name) - demands: - - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker - dependsOn: [] - jobs: - - job: BuildDotnetComponents - displayName: Build Dotnet Components - steps: - - script: scripts/linux/buildBranch.sh -c $(Build.Configuration) --no-rocksdb-bin - name: build - displayName: Build ($(Build.Configuration)) dotnet artifacts - - bash: | - mkdir $(Build.ArtifactStagingDirectory)/core-linux && \ - mv $(Build.BinariesDirectory)/publish/{CACertificates,e2e_deployment_files,e2e_test_files,\ - scripts,IotEdgeQuickstart*,LeafDevice*} $(Build.ArtifactStagingDirectory)/core-linux - displayName: Move/stage test file artifacts - - script: scripts/linux/createArtifactInfo.sh --output-folder '$(Build.ArtifactStagingDirectory)/core-linux' --build-number $(Build.BuildNumber) - displayName: 'Create Artifact Info File' - - task: PublishBuildArtifacts@1 - displayName: Publish core-linux artifact - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/core-linux' - artifactName: 'core-linux' - - task: PublishBuildArtifacts@1 - displayName: Publish Dotnet Artifacts - inputs: - PathtoPublish: '$(Build.BinariesDirectory)/publish' - artifactName: 'dotnet_artifacts' - StoreAsTar: true - - - template: templates/build-broker-watchdog.yaml - - - template: templates/build-api-proxy.yaml - - - template: templates/rust-test-module-build.yaml - parameters: - module.path: 'test/modules/generic-mqtt-tester' - module.name: 'generic-mqtt-tester' - - # This stage first downloads the dotnet and rust artifacts from the previous stage, then - # copies the mqtt and watchdog artifacts into the appropriate directories - # required by consolidate-build-artifacts.sh. It then runs consolidate-build-artifacts.sh, - # which moves the EH dockerfile, edge-hub executable, watchdog executable, and mqttd - # executable into the appropriate directories for later building the edge-hub image in the - # 'Build Images' stage of this pipeline. Finally, this stage consolidates the aforementioned - # EH artifacts with the other dotnet artifacts and publishes them all in an artifact named - # 'consolidated_artifacts'. -################################################################################ -- stage: ConsolidateAndPubishArtifacts -################################################################################ - displayName: Consolidate And Publish Artifacts - pool: - name: $(pool.linux.name) - demands: - - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker - dependsOn: BuildExecutables - jobs: - - job: ConsolidateAndPublishArtifacts - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Artifacts - inputs: - buildType: 'current' - downloadPath: '$(Build.ArtifactStagingDirectory)' - downloadType: specific - extractTars: true - - - task: CopyFiles@2 - displayName: Copy mqttd (amd64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_amd64' - contents: 'mqttd' - targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/x86_64-unknown-linux-musl/release' - - - task: CopyFiles@2 - displayName: Copy mqttd (arm32) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_arm32' - contents: 'mqttd' - targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/armv7-unknown-linux-gnueabihf/release' - - - task: CopyFiles@2 - displayName: Copy mqttd (arm64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_arm64' - contents: 'mqttd' - targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/aarch64-unknown-linux-gnu/release' - - - task: CopyFiles@2 - displayName: Copy watchdog (amd64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_amd64' - contents: 'watchdog' - targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/x86_64-unknown-linux-musl/release' - - - task: CopyFiles@2 - displayName: Copy watchdog (arm32) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_arm32' - contents: 'watchdog' - targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/armv7-unknown-linux-gnueabihf/release' - - - task: CopyFiles@2 - displayName: Copy watchdog (arm64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_arm64' - contents: 'watchdog' - targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/aarch64-unknown-linux-gnu/release' - - script: scripts/linux/consolidate-build-artifacts.sh --dotnet-artifacts-source-dir '$(Agent.TempDirectory)/extracted_tars/dotnet_artifacts' --rust-artifacts-source-dir '$(Build.ArtifactStagingDirectory)' --dest-dir '$(Build.BinariesDirectory)/publish' --artifact-name 'edge-hub' - displayName: Consolidate Edge Hub Artifacts - - # TODO: Consider moving the following copies into consolidate-build-artifacts.sh - - task: CopyFiles@2 - displayName: Copy/Stage dotnet '$(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template' - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/dotnet_artifacts' - contents: '**' - targetFolder: '$(Build.BinariesDirectory)/publish' - - - task: CopyFiles@2 - displayName: Copy API Proxy Artifacts (amd64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_x86_64' - contents: '**' - targetFolder: '$(Build.BinariesDirectory)/publish' - - - task: CopyFiles@2 - displayName: Copy API Proxy Artifacts (arm32) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_armv7l' - contents: '**' - targetFolder: '$(Build.BinariesDirectory)/publish' - - - task: CopyFiles@2 - displayName: Copy API Proxy Artifacts (arm64) - inputs: - sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_aarch64' - contents: '**' - targetFolder: '$(Build.BinariesDirectory)/publish' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifacts' - inputs: - pathToPublish: '$(Build.BinariesDirectory)/publish' - artifactName: 'consolidated_artifacts' - StoreAsTar: true - -################################################################################ -- stage: BuildImages -################################################################################ - displayName: Build Images - pool: - name: $(pool.linux.name) - demands: - - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker - dependsOn: ConsolidateAndPubishArtifacts - jobs: - - job: BuildImageEdgeAgent - steps: - - template: templates/image-linux.yaml - parameters: - name: Edge Agent - imageName: azureiotedge-agent - project: Microsoft.Azure.Devices.Edge.Agent.Service - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageEdgeHub - steps: - - template: templates/image-linux.yaml - parameters: - name: Edge Hub - imageName: azureiotedge-hub - project: edge-hub - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageTemperatureSensor - steps: - - template: templates/image-linux.yaml - parameters: - name: Temperature Sensor - imageName: azureiotedge-simulated-temperature-sensor - project: SimulatedTemperatureSensor - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageDiagnostics - steps: - - template: templates/image-linux.yaml - parameters: - name: Diagnostics - imageName: azureiotedge-diagnostics - project: IotedgeDiagnosticsDotnet - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageTemperatureFilter - steps: - - template: templates/image-linux.yaml - parameters: - name: Temperature Filter - imageName: azureiotedge-temperature-filter - project: TemperatureFilter - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageLoadGen - steps: - - template: templates/image-linux.yaml - parameters: - name: Load Gen - imageName: azureiotedge-load-gen - project: load-gen - bin_dir: '$(Build.BinariesDirectory)' - - job: BuildImageTestAnalyzer - steps: - - template: templates/image-linux.yaml - parameters: - name: Test Analyzer - imageName: azureiotedge-analyzer - project: TestAnalyzer - bin_dir: '$(Build.BinariesDirectory)' - - # Functions Sample - Not supported on linux arm64 - - job: BuildImageFunctionsSample - steps: - - template: templates/image-linux.yaml - parameters: - name: Functions Sample - imageName: azureiotedge-functions-filter - project: EdgeHubTriggerCSharp - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageDirectMethodSender - steps: - - template: templates/image-linux.yaml - parameters: - name: Direct Method Sender - imageName: azureiotedge-direct-method-sender - project: DirectMethodSender - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageDirectMethodReceiver - steps: - - template: templates/image-linux.yaml - parameters: - name: Direct Method Receiver - imageName: azureiotedge-direct-method-receiver - project: DirectMethodReceiver - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageMetricsValidator - steps: - - template: templates/image-linux.yaml - parameters: - name: Metrics Validator - imageName: azureiotedge-metrics-validator - project: MetricsValidator - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageNumberLogger - steps: - - template: templates/image-linux.yaml - parameters: - name: Number Logger - imageName: azureiotedge-number-logger - project: NumberLogger - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageModuleRestarter - steps: - - template: templates/image-linux.yaml - parameters: - name: Module Restarter - imageName: azureiotedge-module-restarter - project: ModuleRestarter - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageTwinTester - steps: - - template: templates/image-linux.yaml - parameters: - name: TwinTester - imageName: azureiotedge-twin-tester - project: TwinTester - bin_dir: '$(Build.BinariesDirectory)' - - job: BuildImageRelayer - steps: - - template: templates/image-linux.yaml - parameters: - name: Relayer - imageName: azureiotedge-relayer - project: Relayer - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageNetworkController - steps: - - template: templates/image-linux.yaml - parameters: - name: Network Controller - imageName: azureiotedge-network-controller - project: NetworkController - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageTestResultCoordinator - steps: - - template: templates/image-linux.yaml - parameters: - name: TestResultCoordinator - imageName: azureiotedge-test-result-coordinator - project: TestResultCoordinator - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageMetricsCollector - steps: - - template: templates/image-linux.yaml - parameters: - name: Metrics Collector - imageName: azureiotedge-metrics-collector - project: MetricsCollector - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageDeploymentTester - steps: - - template: templates/image-linux.yaml - parameters: - name: DeploymentTester - imageName: azureiotedge-deployment-tester - project: DeploymentTester - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageEdgeHubRestartTester - steps: - - template: templates/image-linux.yaml - parameters: - name: EdgeHubRestartTester - imageName: azureiotedge-edgehub-restart-tester - project: EdgeHubRestartTester - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageCloudToDeviceMessageTester - steps: - - template: templates/image-linux.yaml - parameters: - name: Cloud To Device Message Tester - imageName: azureiotedge-c2dmessage-tester - project: CloudToDeviceMessageTester - bin_dir: '$(Build.BinariesDirectory)' - - - job: BuildImageApiProxy - steps: - - bash: | - sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static && \ - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes && \ - docker buildx rm mbuilder || true && \ - docker buildx create --name mbuilder || true && \ - docker buildx use mbuilder - docker -v - - template: templates/image-linux.yaml - parameters: - name: API Proxy - imageName: azureiotedge-api-proxy - project: api-proxy-module - buildx_flag: false - bin_dir: '$(Build.BinariesDirectory)' - - # Rust test modules - - job: BuildRustImage - displayName: Build Rust Image - strategy: - matrix: - GenericMqttTester: - moduleName: generic-mqtt-tester - modulePath: test/modules/generic-mqtt-tester - steps: - - template: templates/rust-test-module-images.yaml - parameters: - module.name: $(moduleName) - module.path: $(modulePath) - -################################################################################ -- stage: PublishManifests -################################################################################ - displayName: Publish Manifests - pool: - name: $(pool.linux.name) - demands: - - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker - dependsOn: BuildImages - variables: - tags: "['latest']" - jobs: - - job: PublishManifest - displayName: Publish Manifest - strategy: - matrix: - EdgeAgent: - manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-agent/docker/manifest.yaml.template' - EdgeHub: - manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-hub/docker/manifest.yaml.template' - SimulatedTemperatureSensor: - manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template' - Diagnostics: - manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/iotedge-diagnostics-dotnet/docker/manifest.yaml.template' - ApiProxy: - manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template' - steps: - - task: Docker@2 - displayName: Docker login edgebuilds - inputs: - command: login - containerRegistry: iotedge-edgebuilds-acr - - - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(Build.BuildNumber) -t $(manifestFilePath) -n microsoft --tags "$(tags)" - - - job: ComponentGovernance - steps: - - task: ComponentGovernanceComponentDetection@0 - inputs: - dockerImagesToScan: '$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-amd64,$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-arm32v7,$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-arm64v8' \ No newline at end of file + - template: templates/build-images.yaml \ No newline at end of file diff --git a/builds/misc/packages.yaml b/builds/misc/packages.yaml index cb8f0ed08be..f8ac4fe6159 100644 --- a/builds/misc/packages.yaml +++ b/builds/misc/packages.yaml @@ -5,109 +5,6 @@ trigger: - master - release/* pr: none -jobs: -################################################################################ - - job: linux -################################################################################ - displayName: Linux - pool: - name: $(pool.linux.name) - demands: - - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker - strategy: - matrix: - Centos75-amd64: - arch: amd64 - os: centos7 - target.iotedged: edgelet/target/rpmbuild/RPMS/x86_64 - Debian9-amd64: - os: debian9 - arch: amd64 - target.iotedged: edgelet/target/release - Debian9-arm32v7: - os: debian9 - arch: arm32v7 - target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release - Debian9-aarch64: - os: debian9 - arch: aarch64 - target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release - - Debian10-amd64: - os: debian10 - arch: amd64 - target.iotedged: edgelet/target/release - Debian10-arm32v7: - os: debian10 - arch: arm32v7 - target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release - Debian10-aarch64: - os: debian10 - arch: aarch64 - target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release - - Debian11-amd64: - os: debian11 - arch: amd64 - target.iotedged: edgelet/target/release - Debian11-arm32v7: - os: debian11 - arch: arm32v7 - target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release - Debian11-aarch64: - os: debian11 - arch: aarch64 - target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release - - Ubuntu1804-amd64: - os: ubuntu18.04 - arch: amd64 - target.iotedged: edgelet/target/release - Ubuntu1804-arm32v7: - os: ubuntu18.04 - arch: arm32v7 - target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release - Ubuntu1804-aarch64: - os: ubuntu18.04 - arch: aarch64 - target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release - - Ubuntu2004-amd64: - arch: amd64 - os: ubuntu20.04 - target.iotedged: edgelet/target/release - Ubuntu2004-arm32v7: - arch: arm32v7 - os: ubuntu20.04 - target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release - Ubuntu2004-aarch64: - arch: aarch64 - os: ubuntu20.04 - target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release - steps: - - bash: | - BASE_VERSION=`cat $BUILD_SOURCESDIRECTORY/edgelet/version.txt` - VERSION="$BASE_VERSION$BUILD_BUILDNUMBER" - echo "Version: $VERSION" - echo "##vso[task.setvariable variable=VERSION;]$VERSION" - - echo "##vso[task.setvariable variable=PACKAGE_ARCH;]$(arch)" - echo "##vso[task.setvariable variable=PACKAGE_OS;]$(os)" - displayName: Set Version - - script: edgelet/build/linux/package.sh - displayName: Create iotedged packages - - task: CopyFiles@2 - displayName: Copy iotedged Files to Artifact Staging - inputs: - SourceFolder: $(target.iotedged) - Contents: | - *.deb - *.rpm - TargetFolder: '$(build.artifactstagingdirectory)' - - task: PublishBuildArtifacts@1 - displayName: Publish Artifacts - inputs: - PathtoPublish: '$(build.artifactstagingdirectory)' - ArtifactName: 'iotedged-$(os)-$(arch)' - condition: succeededOrFailed() +stages: + - template: templates/build-packages.yaml \ No newline at end of file diff --git a/builds/misc/templates/build-images.yaml b/builds/misc/templates/build-images.yaml new file mode 100644 index 00000000000..c90c27ff251 --- /dev/null +++ b/builds/misc/templates/build-images.yaml @@ -0,0 +1,456 @@ +parameters: + - name: 'E2EBuild' + type: boolean + default: false + +stages: + + ################################################################################ + - stage: CheckBuildImages + ################################################################################ + displayName: Check For Source Code Changes + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: [] + jobs: + - job: check_source_change_runtime + displayName: Check Source Changes Edgelet (changes are outside of test or doc folders) + steps: + - bash: | + git log -m -1 --name-only --first-parent --pretty="" | egrep -i -v '^(test/Microsoft.Azure.Devices.Edge.Test|doc|edgelet)' + if [[ $? == 0 ]]; then + echo "Detected changes outside of Test Folders" + echo "##vso[task.setvariable variable=RUNTIMECHANGES;isOutput=true]TRUE" + fi + displayName: Check changes in runtime sources + name: check_files + + ################################################################################ + - stage: BuildExecutables + ################################################################################ + displayName: Build Excecutables + condition: | + or + ( + eq(${{ parameters['E2EBuild'] }}, false), + eq(dependencies.CheckBuildImages.outputs['check_source_change_runtime.check_files.RUNTIMECHANGES'], 'true') + ) + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: CheckBuildImages + jobs: + - job: BuildDotnetComponents + displayName: Build Dotnet Components + steps: + - script: scripts/linux/buildBranch.sh -c $(Build.Configuration) --no-rocksdb-bin + name: build + displayName: Build ($(Build.Configuration)) dotnet artifacts + - bash: | + mkdir $(Build.ArtifactStagingDirectory)/core-linux && \ + mv $(Build.BinariesDirectory)/publish/{CACertificates,e2e_deployment_files,e2e_test_files,\ + scripts,IotEdgeQuickstart*,LeafDevice*} $(Build.ArtifactStagingDirectory)/core-linux + displayName: Move/stage test file artifacts + - script: scripts/linux/createArtifactInfo.sh --output-folder '$(Build.ArtifactStagingDirectory)/core-linux' --build-number $(Build.BuildNumber) + displayName: 'Create Artifact Info File' + - task: PublishBuildArtifacts@1 + displayName: Publish core-linux artifact + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/core-linux' + artifactName: 'core-linux' + - task: PublishBuildArtifacts@1 + displayName: Publish Dotnet Artifacts + inputs: + PathtoPublish: '$(Build.BinariesDirectory)/publish' + artifactName: 'dotnet_artifacts' + StoreAsTar: true + + - template: build-broker-watchdog.yaml + + - template: build-api-proxy.yaml + + - template: rust-test-module-build.yaml + parameters: + module.path: 'test/modules/generic-mqtt-tester' + module.name: 'generic-mqtt-tester' + + # This stage first downloads the dotnet and rust artifacts from the previous stage, then + # copies the mqtt and watchdog artifacts into the appropriate directories + # required by consolidate-build-artifacts.sh. It then runs consolidate-build-artifacts.sh, + # which moves the EH dockerfile, edge-hub executable, watchdog executable, and mqttd + # executable into the appropriate directories for later building the edge-hub image in the + # 'Build Images' stage of this pipeline. Finally, this stage consolidates the aforementioned + # EH artifacts with the other dotnet artifacts and publishes them all in an artifact named + # 'consolidated_artifacts'. + ################################################################################ + - stage: ConsolidateAndPubishArtifacts + ################################################################################ + displayName: Consolidate And Publish artifacts + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: BuildExecutables + jobs: + - job: ConsolidateAndPublishArtifacts + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + buildType: 'current' + downloadPath: '$(Build.ArtifactStagingDirectory)' + downloadType: specific + extractTars: true + + - task: CopyFiles@2 + displayName: Copy mqttd (amd64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_amd64' + contents: 'mqttd' + targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/x86_64-unknown-linux-musl/release' + + - task: CopyFiles@2 + displayName: Copy mqttd (arm32) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_arm32' + contents: 'mqttd' + targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/armv7-unknown-linux-gnueabihf/release' + + - task: CopyFiles@2 + displayName: Copy mqttd (arm64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/mqttd_arm64' + contents: 'mqttd' + targetFolder: '$(Build.ArtifactStagingDirectory)/mqtt/target/aarch64-unknown-linux-gnu/release' + + - task: CopyFiles@2 + displayName: Copy watchdog (amd64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_amd64' + contents: 'watchdog' + targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/x86_64-unknown-linux-musl/release' + + - task: CopyFiles@2 + displayName: Copy watchdog (arm32) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_arm32' + contents: 'watchdog' + targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/armv7-unknown-linux-gnueabihf/release' + + - task: CopyFiles@2 + displayName: Copy watchdog (arm64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/watchdog_arm64' + contents: 'watchdog' + targetFolder: '$(Build.ArtifactStagingDirectory)/edge-hub/watchdog/target/aarch64-unknown-linux-gnu/release' + - script: scripts/linux/consolidate-build-artifacts.sh --dotnet-artifacts-source-dir '$(Agent.TempDirectory)/extracted_tars/dotnet_artifacts' --rust-artifacts-source-dir '$(Build.ArtifactStagingDirectory)' --dest-dir '$(Build.BinariesDirectory)/publish' --artifact-name 'edge-hub' + displayName: Consolidate Edge Hub Artifacts + + # TODO: Consider moving the following copies into consolidate-build-artifacts.sh + - task: CopyFiles@2 + displayName: Copy/Stage dotnet '$(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template' + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/dotnet_artifacts' + contents: '**' + targetFolder: '$(Build.BinariesDirectory)/publish' + + - task: CopyFiles@2 + displayName: Copy API Proxy Artifacts (amd64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_x86_64' + contents: '**' + targetFolder: '$(Build.BinariesDirectory)/publish' + + - task: CopyFiles@2 + displayName: Copy API Proxy Artifacts (arm32) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_armv7l' + contents: '**' + targetFolder: '$(Build.BinariesDirectory)/publish' + + - task: CopyFiles@2 + displayName: Copy API Proxy Artifacts (arm64) + inputs: + sourceFolder: '$(Agent.TempDirectory)/extracted_tars/api_proxy_aarch64' + contents: '**' + targetFolder: '$(Build.BinariesDirectory)/publish' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + inputs: + pathToPublish: '$(Build.BinariesDirectory)/publish' + artifactName: 'consolidated_artifacts' + StoreAsTar: true + + ################################################################################ + - stage: BuildImages + ################################################################################ + displayName: Build Images + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: ConsolidateAndPubishArtifacts + jobs: + - job: BuildImageEdgeAgent + steps: + - template: image-linux.yaml + parameters: + name: Edge Agent + imageName: azureiotedge-agent + project: Microsoft.Azure.Devices.Edge.Agent.Service + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageEdgeHub + steps: + - template: image-linux.yaml + parameters: + name: Edge Hub + imageName: azureiotedge-hub + project: edge-hub + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageTemperatureSensor + steps: + - template: image-linux.yaml + parameters: + name: Temperature Sensor + imageName: azureiotedge-simulated-temperature-sensor + project: SimulatedTemperatureSensor + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageDiagnostics + steps: + - template: image-linux.yaml + parameters: + name: Diagnostics + imageName: azureiotedge-diagnostics + project: IotedgeDiagnosticsDotnet + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageTemperatureFilter + steps: + - template: image-linux.yaml + parameters: + name: Temperature Filter + imageName: azureiotedge-temperature-filter + project: TemperatureFilter + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageLoadGen + steps: + - template: image-linux.yaml + parameters: + name: Load Gen + imageName: azureiotedge-load-gen + project: load-gen + bin_dir: '$(Build.BinariesDirectory)' + - job: BuildImageTestAnalyzer + steps: + - template: image-linux.yaml + parameters: + name: Test Analyzer + imageName: azureiotedge-analyzer + project: TestAnalyzer + bin_dir: '$(Build.BinariesDirectory)' + + # Functions Sample - Not supported on linux arm64 + - job: BuildImageFunctionsSample + steps: + - template: image-linux.yaml + parameters: + name: Functions Sample + imageName: azureiotedge-functions-filter + project: EdgeHubTriggerCSharp + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageDirectMethodSender + steps: + - template: image-linux.yaml + parameters: + name: Direct Method Sender + imageName: azureiotedge-direct-method-sender + project: DirectMethodSender + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageDirectMethodReceiver + steps: + - template: image-linux.yaml + parameters: + name: Direct Method Receiver + imageName: azureiotedge-direct-method-receiver + project: DirectMethodReceiver + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageMetricsValidator + steps: + - template: image-linux.yaml + parameters: + name: Metrics Validator + imageName: azureiotedge-metrics-validator + project: MetricsValidator + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageNumberLogger + steps: + - template: image-linux.yaml + parameters: + name: Number Logger + imageName: azureiotedge-number-logger + project: NumberLogger + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageModuleRestarter + steps: + - template: image-linux.yaml + parameters: + name: Module Restarter + imageName: azureiotedge-module-restarter + project: ModuleRestarter + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageTwinTester + steps: + - template: image-linux.yaml + parameters: + name: TwinTester + imageName: azureiotedge-twin-tester + project: TwinTester + bin_dir: '$(Build.BinariesDirectory)' + - job: BuildImageRelayer + steps: + - template: image-linux.yaml + parameters: + name: Relayer + imageName: azureiotedge-relayer + project: Relayer + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageNetworkController + steps: + - template: image-linux.yaml + parameters: + name: Network Controller + imageName: azureiotedge-network-controller + project: NetworkController + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageTestResultCoordinator + steps: + - template: image-linux.yaml + parameters: + name: TestResultCoordinator + imageName: azureiotedge-test-result-coordinator + project: TestResultCoordinator + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageMetricsCollector + steps: + - template: image-linux.yaml + parameters: + name: Metrics Collector + imageName: azureiotedge-metrics-collector + project: MetricsCollector + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageDeploymentTester + steps: + - template: image-linux.yaml + parameters: + name: DeploymentTester + imageName: azureiotedge-deployment-tester + project: DeploymentTester + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageEdgeHubRestartTester + steps: + - template: image-linux.yaml + parameters: + name: EdgeHubRestartTester + imageName: azureiotedge-edgehub-restart-tester + project: EdgeHubRestartTester + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageCloudToDeviceMessageTester + steps: + - template: image-linux.yaml + parameters: + name: Cloud To Device Message Tester + imageName: azureiotedge-c2dmessage-tester + project: CloudToDeviceMessageTester + bin_dir: '$(Build.BinariesDirectory)' + + - job: BuildImageApiProxy + steps: + - bash: | + sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static && \ + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes && \ + docker buildx rm mbuilder || true && \ + docker buildx create --name mbuilder || true && \ + docker buildx use mbuilder + docker -v + - template: image-linux.yaml + parameters: + name: API Proxy + imageName: azureiotedge-api-proxy + project: api-proxy-module + buildx_flag: false + bin_dir: '$(Build.BinariesDirectory)' + + # Rust test modules + - job: BuildRustImage + displayName: Build Rust Image + strategy: + matrix: + GenericMqttTester: + moduleName: generic-mqtt-tester + modulePath: test/modules/generic-mqtt-tester + steps: + - template: rust-test-module-images.yaml + parameters: + module.name: $(moduleName) + module.path: $(modulePath) + + ################################################################################ + - stage: PublishManifests + ################################################################################ + displayName: Publish Manifests + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: BuildImages + variables: + tags: "['latest']" + jobs: + - job: PublishManifest + displayName: Publish Manifest + strategy: + matrix: + EdgeAgent: + manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-agent/docker/manifest.yaml.template' + EdgeHub: + manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-hub/docker/manifest.yaml.template' + SimulatedTemperatureSensor: + manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template' + Diagnostics: + manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/iotedge-diagnostics-dotnet/docker/manifest.yaml.template' + ApiProxy: + manifestFilePath: '$(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template' + steps: + - task: Docker@2 + displayName: Docker login edgebuilds + inputs: + command: login + containerRegistry: iotedge-edgebuilds-acr + + - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(Build.BuildNumber) -t $(manifestFilePath) -n microsoft --tags "$(tags)" + + - job: ComponentGovernance + steps: + - task: ComponentGovernanceComponentDetection@0 + inputs: + dockerImagesToScan: '$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-amd64,$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-arm32v7,$(registry.address)/microsoft/azureiotedge-api-proxy:$(Build.BuildNumber)-linux-arm64v8' \ No newline at end of file diff --git a/builds/misc/templates/build-packages.yaml b/builds/misc/templates/build-packages.yaml new file mode 100644 index 00000000000..ae148647874 --- /dev/null +++ b/builds/misc/templates/build-packages.yaml @@ -0,0 +1,145 @@ +parameters: + - name: 'E2EBuild' + type: boolean + default: false + +stages: + + ################################################################################ + - stage: CheckBuildPackages + ################################################################################ + displayName: Check For Source Code Changes + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: [] + jobs: + - job: check_source_change_edgelet + displayName: Check Source Changes Edgelet (changes ARE in builds or edgelet) + steps: + - bash: | + git log -m -1 --name-only --first-parent --pretty="" | egrep -i '^(builds|edgelet)' + if [[ $? == 0 ]]; then + echo "Detected changes inside builds or edgelet folders" + echo "##vso[task.setvariable variable=EDGELETCHANGES;isOutput=true]TRUE" + fi + displayName: Check changes in edgelet sources + name: check_files + + ################################################################################ + - stage: BuildPackages + ################################################################################ + displayName: Build Packages + condition: | + or + ( + eq(${{ parameters['E2EBuild'] }}, false), + eq(dependencies.CheckBuildPackages.outputs['check_source_change_edgelet.check_files.EDGELETCHANGES'], 'true') + ) + pool: + name: $(pool.linux.name) + demands: + - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker + dependsOn: CheckBuildPackages + jobs: + - job: linux + displayName: Linux + strategy: + matrix: + Centos75-amd64: + arch: amd64 + os: centos7 + target.iotedged: edgelet/target/rpmbuild/RPMS/x86_64 + Debian9-amd64: + os: debian9 + arch: amd64 + target.iotedged: edgelet/target/release + Debian9-arm32v7: + os: debian9 + arch: arm32v7 + target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release + Debian9-aarch64: + os: debian9 + arch: aarch64 + target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release + + Debian10-amd64: + os: debian10 + arch: amd64 + target.iotedged: edgelet/target/release + Debian10-arm32v7: + os: debian10 + arch: arm32v7 + target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release + Debian10-aarch64: + os: debian10 + arch: aarch64 + target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release + Debian11-amd64: + os: debian11 + arch: amd64 + target.iotedged: edgelet/target/release + Debian11-arm32v7: + os: debian11 + arch: arm32v7 + target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release + Debian11-aarch64: + os: debian11 + arch: aarch64 + target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release + + Ubuntu1804-amd64: + os: ubuntu18.04 + arch: amd64 + target.iotedged: edgelet/target/release + Ubuntu1804-arm32v7: + os: ubuntu18.04 + arch: arm32v7 + target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release + Ubuntu1804-aarch64: + os: ubuntu18.04 + arch: aarch64 + target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release + + Ubuntu2004-amd64: + arch: amd64 + os: ubuntu20.04 + target.iotedged: edgelet/target/release + Ubuntu2004-arm32v7: + arch: arm32v7 + os: ubuntu20.04 + target.iotedged: edgelet/target/armv7-unknown-linux-gnueabihf/release + Ubuntu2004-aarch64: + arch: aarch64 + os: ubuntu20.04 + target.iotedged: edgelet/target/aarch64-unknown-linux-gnu/release + steps: + - bash: | + BASE_VERSION=`cat $BUILD_SOURCESDIRECTORY/edgelet/version.txt` + VERSION="$BASE_VERSION$BUILD_BUILDNUMBER" + echo "Version: $VERSION" + echo "##vso[task.setvariable variable=VERSION;]$VERSION" + + echo "##vso[task.setvariable variable=PACKAGE_ARCH;]$(arch)" + echo "##vso[task.setvariable variable=PACKAGE_OS;]$(os)" + displayName: Set Version + condition: or(eq(${{ parameters['E2EBuild'] }}, false), eq(variables.arch,'amd64')) + - script: edgelet/build/linux/package.sh + displayName: Create iotedged packages + condition: or(eq(${{ parameters['E2EBuild'] }}, false), eq(variables.arch,'amd64')) + - task: CopyFiles@2 + displayName: Copy iotedged Files to Artifact Staging + inputs: + SourceFolder: $(target.iotedged) + Contents: | + *.deb + *.rpm + TargetFolder: '$(build.artifactstagingdirectory)' + condition: or(eq(${{ parameters['E2EBuild'] }}, false), eq(variables.arch,'amd64')) + - task: PublishBuildArtifacts@1 + displayName: Publish Artifacts + inputs: + PathtoPublish: '$(build.artifactstagingdirectory)' + ArtifactName: 'iotedged-$(os)-$(arch)' + condition: or(eq(${{ parameters['E2EBuild'] }}, false), eq(variables.arch,'amd64')) \ No newline at end of file