azure-pipelines.yml

trigger:
- master

#schedules:
#- cron: "30 2 */3 * *"
#  displayName: Daily early morning run
#  branches:
#    include:
#    - master
#  always: true

variables:
  vmImageName: 'ubuntu-18.04'

pool:
  vmImage: $(vmImageName)

#container:
#    image: ubuntu:latest
    # options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
# fix for sudo
#  steps:
#    - script: |
#        /tmp/docker exec -t -u 0 ci-container \
#        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
#      displayName: Set up sudo

jobs:
### TEST JOB ###
  - job: Test
    displayName: Test job only
    condition: eq(variables['runTestJob'], 'true')
    variables:
      ramdisk: $(System.DefaultWorkingDirectory)/ramdisk
    steps: 

    # - checkout: none
    - script: |
        echo "largest files on /"
        #sudo find / -xdev -type f -exec du -Sh {} + | sort -rh | head -n 15
        echo "directories:"
        #sudo du -Shx / | sort -rh | head -15
        #sudo dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -rn | head -n 20
        #sudo dpkg --purge libboost-dev default-jre-headless swift google-chrome-stable google-cloud-sdk hhvm zulu-11-azure-jdk zulu-12-azure-jdk ghc openjdk-11-jre-headless
        #sudo apt autoremove
        #sudo apt clean
        df -h
        #sudo snap remove dotnet-sdk julia
        df -h
        pwd
        ls -lah
        sudo touch /mnt/foobar || true
        sudo mkdir /mnt/user_tmp || true
        ls -lah /mnt/user_tmp || true
        sudo chown vsts /mnt/user_tmp || true
        touch /mnt/user_tmp/foobar || true
        ls -lah /mnt/user_tmp/ || true
        find /mnt
        echo $(System.DefaultWorkingDirectory)
        echo $(Pipeline.Workspace)
        mkdir $(ramdisk)
        sudo mount -t tmpfs -o rw,size=3G tmpfs $(ramdisk)
        ls -lah $(ramdisk)
        sudo chmod 777 $(ramdisk)
        touch $(ramdisk)/foobar
        ls -lah $(ramdisk)/foobar
        mount 
        sudo umount $(ramdisk)
        echo \"foo\" | tr -d '"'
        echo $(Agent.Name)
        echo $(Agent.HomeDirectory)
        echo $(Agent.BuildDirectory)
        ls -la /home/vsts
        env | grep PATH
        env | grep HOME
        # bash scripts/addpeers_2.0.sh
        free -m
        # sudo mount -t tmpfs -o rw,size=2G tmpfs ./ramdisk
        # sudo chmod 777 ./ramdisk
        # sudo find /lib/modules/$(uname -r) -type f -name '*.ko'
        # sudo find /lib/modules/$(uname -r) -type f -name '*.ko*'
        # sudo depmod -av|grep f2fs
        uname -r
      displayName: Test commands

    - task: GithubRelease@0
      displayName: 'Create GitHub Release for snapshots'
      inputs:
        isPreRelease: true
        title: 'Test Release'
        gitHubConnection: phlogi-personal
        repositoryName: phlogi/tezos-snapshots
        tagSource: manual
        releaseNotesSource: input
        releaseNotes: 'bla bla'
        addChangeLog: false
        tag: $(Build.BuildNumber)
        assets: |
            /home/vsts/work/1/s/**

### JOB ###
  - job: Build
    displayName: Build and publish tezos binaries
    #condition: and(always(), eq(variables['Build.Reason'], 'Schedule'))
    variables:
      tezosLastCommit: unknown
      tezosLastCommitStamp: unknown
      recentCommitUpdate: false
    steps:
    - checkout: none

    - task: PowerShell@2
      displayName: Set date time stamp variable
      condition: false
      inputs:
        targetType: 'inline'
        script: |
          $Time = Get-Date
          $Time = $Time.ToUniversalTime()
          $date = $Time.ToString("yyyy-MM-dd HH:mmK")
          Write-Host "##vso[task.setvariable variable=datetimestamp]$date"
          $UnixStamp = Get-Date -UFormat %s
          Write-Host "##vso[task.setvariable variable=unixdatetimestamp]$UnixStamp"

    - script: |
        git clone -b $(gitCheckoutBranch) --depth 1 $(gitCloneUrl)
        cd tezos
        commit_id=$(git log --format="%h" -n 1) && echo $commit_id
        stamp=$(git log -1 --format=%cd -n 1) && echo $stamp
        unix_git_stamp=$(git log -1 --format=%ct -n 1)
        echo "##vso[task.setvariable variable=tezosLastCommit]$commit_id"
        echo "##vso[task.setvariable variable=tezosLastCommitStamp]$stamp"
        current_unix_stamp=$(date +"%s")
        need_update=false
        if (( current_unix_stamp-unix_git_stamp < $(lookbackDurationCommitUpdate) )); then need_update=true && echo "found recent commit, shall build"; fi
        [[ $(forceBuild) == true ]] && need_update=true
        echo "##vso[task.setvariable variable=recentCommitUpdate]$need_update"
      displayName: Checkout tezos

# note: variables are not available immediately within the same script block after being set with task.setvariable
    - script: sudo add-apt-repository ppa:avsm/ppa && sudo apt-get update && sudo apt-get remove -y --purge man-db && sudo apt-get install -yy -qq opam rsync m4 build-essential patch unzip bubblewrap wget libev-dev libgmp-dev libhidapi-dev m4 perl pkg-config
      displayName: Install opam and dependencies
      condition: eq(variables['recentCommitUpdate'], true)

    - script: |
        opam init --bare -n
        eval $(opam env)
        cd tezos && ls -la && make build-deps && eval $(opam env) && make && ls -lah tezos-* && mkdir ../bin && mv -v tezos-* ../bin/
      displayName: Build tezos
      condition: eq(variables['recentCommitUpdate'], true)

    - publish: '$(System.DefaultWorkingDirectory)/bin/'
      artifact: tezos-binaries
      displayName: Publish tezos binaries
      condition: eq(variables['recentCommitUpdate'], true)

    - task: UniversalPackages@0
      displayName: Publish tezos binaries as universal package
      condition: eq(variables['recentCommitUpdate'], true)
      continueOnError: true 
      inputs:
        command: publish
        publishDirectory: '$(System.DefaultWorkingDirectory)/bin/'
        vstsFeedPublish: 'tezos-binaries'
        vstsFeedPackagePublish: 'tezos-binaries-ubuntu-azure'
        packagePublishDescription: 'Tezos binaries [$(tezosLastCommitStamp) / $(tezosLastCommit)] built on $(datetimestamp) within Azure Pipeline running on $(vmImageName).'

    # we should have new binaries here and save those as an artifact. 
    # in order to find them later we save this specific build number into a config artifact
    # this will be used to find the right build having an artifact with the binaries
    - script: |
        echo $(Build.BuildId)
        echo $(Build.BuildId) > $(System.DefaultWorkingDirectory)/build.var
      displayName: Save current build id to file
      condition: eq(variables['recentCommitUpdate'], true)

    - publish: '$(System.DefaultWorkingDirectory)/build.var'
      artifact: pipeline-config
      displayName: Save pipeline-config
      condition: eq(variables['recentCommitUpdate'], true)


    - task: DownloadPipelineArtifact@2
      displayName: Download pipeline config
      condition: eq(variables['recentCommitUpdate'], false)
      inputs:
        buildType: 'specific'
        project: '$(System.TeamProjectId)'
        definition: '$(System.DefinitionId)'
        buildVersionToDownload: 'latest'
        allowPartiallySucceededBuilds: true
        # Needed to bootstrap. 
        allowFailedBuilds: true
        artifactName: 'pipeline-config'
        targetPath: '$(System.DefaultWorkingDirectory)'

      # if no build, save the pipeline-config
    - publish: '$(System.DefaultWorkingDirectory)/build.var'
      artifact: pipeline-config
      displayName: Save pipeline-config
      condition: eq(variables['recentCommitUpdate'], false)

    - script: |
        docker pull tezos/tezos
        docker create -ti --name tz tezos/tezos:mainnet
        mkdir ./bin && pwd
        docker cp tz:/usr/local/bin/tezos-node ./bin/
        ls bin -la
        docker rm -f tz
      displayName: Get tezos-node binary from container image
      condition: false

### JOB ###
  - job: getUpdatedSnapshot
    #condition: and(always(), eq(variables['Build.Reason'], 'Schedule'))
    timeoutInMinutes: 360
    displayName: Download, import snapshot, setup node and run
    variables: 
      snapshotFileName: snapshot.bootstrap
      snapshotsDir: /mnt/user_tmp/snapshots/
      ramdisk: $(System.DefaultWorkingDirectory)/ramdisk
    steps:

    - script: |
        sudo dpkg --purge $(removePackages)
        sudo apt autoremove
        sudo apt clean
        sudo rm -rf $(removeDirectories)
        df -h
        sudo mkdir /mnt/user_tmp
        sudo chown vsts /mnt/user_tmp
        touch /mnt/user_tmp/vsts_can_write_stuff_here.yes
        mkdir '$(snapshotsDir)'
        ls -lah /mnt/user_tmp/
      displayName: Clean VM and setup tmp directory on sdb mounted in /mnt for user vsts

    - task: DownloadPipelineArtifact@2
      displayName: Download pipeline config
      continueOnError: false
      inputs:
        buildType: 'specific'
        project: '$(System.TeamProjectId)'
        definition: '$(System.DefinitionId)'
        buildVersionToDownload: 'latest'
        allowPartiallySucceededBuilds: true
        # Needed to bootstrap
        allowFailedBuilds: true
        artifactName: 'pipeline-config'
        targetPath: '$(System.DefaultWorkingDirectory)'

    - script: |
        ls -lah $(System.DefaultWorkingDirectory)
        cat $(System.DefaultWorkingDirectory)/build.var
        build=$(<$(System.DefaultWorkingDirectory)/build.var)
        echo "##vso[task.setvariable variable=buildNrBinaries]$build"
        echo $build
      displayName: Set build id variable from file
      condition: eq(variables['overrideBuildId'], false)

    - script: sudo apt-get update && sudo apt-get install -yy -qq jq dnsutils libev-dev libgmp-dev libhidapi-dev
      displayName: Install packages

    - task: DownloadPipelineArtifact@2
      displayName: Download tezos binaries artifact 
      continueOnError: false
      inputs:
        buildType: 'specific'
        runId: $(buildNrBinaries)
        project: '$(System.TeamProjectId)'
        definition: '$(System.DefinitionId)'
        buildVersionToDownload: 'specific'
        allowPartiallySucceededBuilds: true
        allowFailedBuilds: true
        artifactName: 'tezos-binaries'
        path: '$(System.DefaultWorkingDirectory)/bin'

    - script: |
        curl -s https://api.github.com/repos/Phlogi/tezos-snapshots/releases/latest | jq -r ".assets[] | select(.name) | .browser_download_url" | grep full | xargs wget -q
      displayName: Download snapshot
      condition: eq(variables['useExternalSnapshot'], true)

    - task: DownloadPipelineArtifact@2
      displayName: Download full snapshot from previous build
      condition: eq(variables['useExternalSnapshot'], false)
      inputs:
        buildType: 'specific'
        artifact: 'snapshots-for-pipeline'
        targetPath: $(Pipeline.Workspace)/snapshots-for-pipeline
        definition: $(System.DefinitionId)
        project: $(System.TeamProjectId)
        # pipelineId: 12 # the build id from which to download the artifacts. For example: 1764
        buildVersionToDownload: 'latest'
        # mv -v $(Pipeline.Workspace)/snapshots-for-pipeline/snapshot-from-build.full.xz $(snapshotFileName).xz

    - script: |
        ls -lah
        cat *full*.xz* > $(snapshotsDir)$(snapshotFileName).xz
        df -h
        rm -v *full*.xz*
        ls -lah $(snapshotsDir)$(snapshotFileName).xz
        latest_block=$(curl -s https://api.github.com/repos/Phlogi/tezos-snapshots/releases/latest | jq -r ".assets[] | select(.name) | .name" | grep full | awk -F '.' '{print $4; exit}')
        echo "##vso[task.setvariable variable=importBlock]$latest_block"
      displayName: Prepare snapshot
      condition: eq(variables['useExternalSnapshot'], true)

    - script: |
        xz -l $(snapshotsDir)$(snapshotFileName).xz
        sb=$(xz --robot -l $(snapshotsDir)$(snapshotFileName).xz | grep file |  awk '{print $5}')
        let sb+=1024
        mkdir $(ramdisk)
        free -m
        sudo mount -t tmpfs -o rw,size=$sb tmpfs $(ramdisk)
        sudo chmod 777 $(ramdisk)
        sudo mount | grep $(ramdisk)
        free -m
        xz $(snapshotsDir)$(snapshotFileName).xz -d -c > $(ramdisk)/$(snapshotFileName) && rm $(snapshotsDir)$(snapshotFileName).xz && ls -lah $(ramdisk)/$(snapshotFileName)
      displayName: Extract snapshot into ramdisk
   
    # universal package seems broken
    - task: UniversalPackages@0
      displayName: 'Download latest tezos binaries'
      condition: false
      inputs:
        command: download
        vstsFeed: 'tezos-binaries'
        vstsFeedPackage: 'tezos-binaries-ubuntu-azure'
        vstsPackageVersion: '*' # = latest version
        downloadDirectory: '$(System.DefaultWorkingDirectory)/bin'
  
    - script: |
        chmod +x ./bin/*

        [[ ! -z "$(importBlock)" ]] && _block_arg="--block=$(importBlock)"
        echo "./bin/tezos-node snapshot import $(ramdisk)/$(snapshotFileName) $(tezosNodeImportOptions) ${_block_arg}"
        ./bin/tezos-node snapshot import $(ramdisk)/$(snapshotFileName) $(tezosNodeImportOptions) ${_block_arg}
      displayName: Import snapshot

    - script: free -m && sudo umount $(ramdisk) && mount | grep tmpfs && free -m && df -h
      displayName: Remove ramdisk

    - task: DownloadPipelineArtifact@2
      displayName: Download tezos config from previous build
      continueOnError: true
      inputs:
        buildType: 'specific'
        allowPartiallySucceededBuilds: true
        artifact: 'node-config'
        targetPath: $(Pipeline.Workspace)/node-config
        #definition: $(System.DefinitionId)
        #project: $(System.TeamProjectId)
        # pipelineId: 12 # the build id from which to download the artifacts. For example: 1764
        buildVersionToDownload: 'latest'

    - script: | 
        ls -la $(Pipeline.Workspace)/snapshots-for-pipeline || true
        ls -la $(Pipeline.Workspace)/node-config || true
      displayName: '[INFO] List downloaded artifacts in workspace folder'

    # move existing config if available, then set condition for this task below
    - script: |
        [[ -f $(Pipeline.Workspace)/node-config/identity.json ]] && mv -v $(Pipeline.Workspace)/node-config/identity.json /home/vsts/.tezos-node/
        [[ ! -f /home/vsts/.tezos-node/identity.json ]] && ./bin/tezos-node identity generate
        ./bin/tezos-node config init
        # copy back stuff from previous build
        [[ -f $(Pipeline.Workspace)/node-config/peers.json ]] && mv -v $(Pipeline.Workspace)/node-config/peers.json /home/vsts/.tezos-node/
        # add official peers
        export PATH="$PATH:/$(System.DefaultWorkingDirectory)/bin" || true
        echo $PATH
        which tezos-node || true
        bash scripts/config-update-official-nodes.sh || true
      displayName: Setup tezos node, restore config from artifact

    - script: |
        timeout 80 ./bin/tezos-node run $(tezosNodeOptions) || true
      displayName: '[Check] Run tezos node in foreground for 80 seconds'

    - script: |
        nohup ./bin/tezos-node run $(tezosNodeOptions) </dev/null >/dev/null 2>&1 &
      displayName: Run tezos node in background

    - script: |
        ps auxw | grep tezos-node
        du -hs /home/vsts/.tezos-node/ || true
        sleep 10
        export PATH="$PATH:/$(System.DefaultWorkingDirectory)/bin" || true
        bash scripts/addpeers_2.0.sh || true
        echo "Waiting for node to be bootstrapped"
        timeout $(chainBootstrapDurationSeconds) ./bin/tezos-client bootstrapped
        ./bin/tezos-client get timestamp
        du -hs /home/vsts/.tezos-node/ || true
      displayName: Run add peers script, wait for node to be bootstrapped or timeout after 3.5 hours

    - script: |
        sleep $(chainSyncDurationSeconds)
        _query_result=$(./bin/tezos-client rpc get /chains/main/blocks/head~10)
        # get information about the tip of the chain in one call, so it's consistent
        b=$(echo $_query_result | jq '.hash' | tr -d '"')
        l=$(echo $_query_result | jq '.header.level')
        t=$(echo $_query_result | jq '.header.timestamp' | tr -d '"' | sed 's/:/./g')
        echo "##vso[task.setvariable variable=block]$b"
        echo "##vso[task.setvariable variable=blockLevel]$l"
        echo "##vso[task.setvariable variable=blockTimeStamp]$t"
        pkill tezos-node
        sleep 10
        ps auxw | grep tezos-node
        pkill -9 tezos-node || true
      displayName: Extract tip of bootstrapped and synced chain, terminate tezos-node
      name: endsync

    - task: CopyFiles@2
      displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: /home/vsts/.tezos-node/
        Contents: |
          **/*.json
          !/context
          !/store
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - script: ls -la $(Build.ArtifactStagingDirectory)
      displayName: '[INFO] List contents of tezos config artifact directory.'

    - publish: '$(Build.ArtifactStagingDirectory)'
      displayName: 'Publish artifact: Node settings json'
      artifact: node-config

    - script: | 
        echo $(block) && echo $(blockLevel) && echo $(blockTimeStamp)
        mkdir -v $(System.DefaultWorkingDirectory)/results
      displayName: Prepare for snapshot exports

    - task: PowerShell@2
      displayName: Set date time stamp variable
      inputs:
        targetType: 'inline'
        script: |
          $Time = Get-Date
          $Time = $Time.ToUniversalTime()
          $date = $Time.ToString("yyyy-MM-dd HH:mmK")
          Write-Host "##vso[task.setvariable variable=datetimestamp]$date"
  
    - script: |
        echo "##vso[task.setvariable variable=rollingSnapshotFileName]$(System.DefaultWorkingDirectory)/results/mainnet.roll.$(date +%F_%H-%M).$(block).$(blockLevel).chain"
        echo "##vso[task.setvariable variable=fullSnapshotFileName]$(System.DefaultWorkingDirectory)/results/mainnet.full.$(date +%F_%H-%M).$(block).$(blockLevel).chain"
      displayName: Set snapshot filenames

    - script: |
        echo $(rollingSnapshotFileName)
        ./bin/tezos-node snapshot export $(rollingSnapshotFileName) --rolling --block=$(block)
      displayName: Export rolling snapshot

    - script: |
        echo $(fullSnapshotFileName)
        ./bin/tezos-node snapshot export $(fullSnapshotFileName) --block=$(block)
      displayName: Export full snapshot

    - script: |
        jq -jr '.[] | select(.last_established_connection != null) | .last_established_connection | .[0] | "[",.addr,"]:",.port,"\n"' /home/vsts/.tezos-node/peers.json > $(System.DefaultWorkingDirectory)/results/public_peers_tezos_mainnet.list
        tail $(System.DefaultWorkingDirectory)/results/public_peers_tezos_mainnet.list || true
      displayName: Export a list of public peers from peers.json file

    - script: |
        xz -8 -T 0 $(rollingSnapshotFileName)
        xz -l $(rollingSnapshotFileName).xz
        
        xz -8 -T 0 -c < $(fullSnapshotFileName) | split -d -b 2040M - $(fullSnapshotFileName).xz.
        rm -v $(fullSnapshotFileName)
        ls -la $(System.DefaultWorkingDirectory)/results/
        sha256sum $(System.DefaultWorkingDirectory)/results/*.xz* > $(System.DefaultWorkingDirectory)/results/checksums.sha256 || true
        cat $(System.DefaultWorkingDirectory)/results/checksums.sha256 || true
      displayName: Compress and split snapshots

    # Publish both snapshots
    - task: CopyFiles@2
      condition: false
      displayName: Prepare publish rolling and full snapshot to artifact staging
      inputs:
        sourceFolder: '$(snapshotsDir)'
        contents: '**' # recursive and everything, including the .list file
        targetFolder: $(Build.ArtifactStagingDirectory)
        cleanTargetFolder: true
        overWrite: true
        flattenFolders: false 
        preserveTimestamp: false 

    - script: echo $(System.DefaultWorkingDirectory)/results/ && find $(System.DefaultWorkingDirectory)/results/
      displayName: '[INFO] List contents of results directory'

    - task: GithubRelease@0
      displayName: 'Create GitHub Release for snapshots'
      inputs:
        isPreRelease: false
        title: 'Tezos Mainnet Snapshots ($(datetimestamp))'
        gitHubConnection: phlogi-personal
        repositoryName: phlogi/tezos-snapshots
        tagSource: manual
        releaseNotesSource: input
        releaseNotes: Rolling and full snapshots up to level $(blockLevel) at $(blockTimeStamp) and block $(block). Current list of public tezos peers. 
        addChangeLog: false
        tag: $(Build.BuildNumber)
        # directory has the files without subfolders
        assets: |
            $(System.DefaultWorkingDirectory)/results/*