Update Newtonsoft usage

eng/Version.Details.xml

@@ -19,14 +19,14 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22372.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22377.18">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>11672d906390046e77a34b6406d9e02229fd7e45</Sha>
+      <Sha>3fdfc33d9a75aaa48ab6174e3762c9ae3a9b6f57</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="7.0.0-beta.22372.1">
+    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="7.0.0-beta.22316.2" Pinned="true" >
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>11672d906390046e77a34b6406d9e02229fd7e45</Sha>
+      <Sha>ccfe6da198c5f05534863bbb1bff66e830e0c6ab</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Dotnet.Sdk.Internal" Version="7.0.100-rc.1.22374.1">
       <Uri>https://github.com/dotnet/installer</Uri>

eng/Versions.props

@@ -63,6 +63,6 @@
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
     <MicrosoftDotNetRemoteExecutorVersion>7.0.0-beta.22316.2</MicrosoftDotNetRemoteExecutorVersion>
     <cdbsosversion>10.0.18362</cdbsosversion>
-    <NewtonSoftJsonVersion>12.0.2</NewtonSoftJsonVersion>
+    <NewtonSoftJsonVersion>13.0.1</NewtonSoftJsonVersion>
   </PropertyGroup>
 </Project>

eng/common/build.sh

@@ -19,6 +19,9 @@ usage()
   echo "Actions:"
   echo "  --restore                  Restore dependencies (short: -r)"
   echo "  --build                    Build solution (short: -b)"
+  echo "  --source-build             Source-build the solution (short: -sb)"
+  echo "                             Will additionally trigger the following actions: --restore, --build, --pack"
+  echo "                             If --configuration is not set explicitly, will also set it to 'Release'"
   echo "  --rebuild                  Rebuild solution"
   echo "  --test                     Run all unit tests in the solution (short: -t)"
   echo "  --integrationTest          Run all integration tests in the solution"
@@ -55,6 +58,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 restore=false
 build=false
+source_build=false
 rebuild=false
 test=false
 integration_test=false
@@ -73,7 +77,7 @@ exclude_ci_binary_log=false
 pipelines_log=false
 
 projects=''
-configuration='Debug'
+configuration=''
 prepare_machine=false
 verbosity='minimal'
 runtime_source_feed=''
@@ -119,6 +123,12 @@ while [[ $# > 0 ]]; do
     -pack)
       pack=true
       ;;
+    -sourcebuild|-sb)
+      build=true
+      source_build=true
+      restore=true
+      pack=true
+      ;;
     -test|-t)
       test=true
       ;;
@@ -168,6 +178,10 @@ while [[ $# > 0 ]]; do
   shift
 done
 
+if [[ -z "$configuration" ]]; then
+  if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi
+fi
+
 if [[ "$ci" == true ]]; then
   pipelines_log=true
   node_reuse=false
@@ -205,6 +219,7 @@ function Build {
     /p:RepoRoot="$repo_root" \
     /p:Restore=$restore \
     /p:Build=$build \
+    /p:ArcadeBuildFromSource=$source_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \
     /p:Pack=$pack \

eng/common/cross/build-rootfs.sh

@@ -18,7 +18,6 @@ usage()
 
 __CodeName=xenial
 __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-__InitialDir=$PWD
 __BuildArch=arm
 __AlpineArch=armv7
 __FreeBSDArch=arm
@@ -43,7 +42,7 @@ __AlpinePackages+=" libedit"
 # symlinks fixer
 __UbuntuPackages+=" symlinks"
 
-# CoreCLR and CoreFX dependencies
+# runtime dependencies
 __UbuntuPackages+=" libicu-dev"
 __UbuntuPackages+=" liblttng-ust-dev"
 __UbuntuPackages+=" libunwind8-dev"
@@ -54,7 +53,7 @@ __AlpinePackages+=" libunwind-dev"
 __AlpinePackages+=" lttng-ust-dev"
 __AlpinePackages+=" compiler-rt-static"
 
-# CoreFX dependencies
+# runtime libraries' dependencies
 __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
@@ -84,17 +83,18 @@ __IllumosPackages+=" zlib-1.2.11"
 __UbuntuPackages+=" libomp5"
 __UbuntuPackages+=" libomp-dev"
 
+__Keyring=
 __UseMirror=0
 
 __UnprocessedBuildArgs=
 while :; do
-    if [ $# -le 0 ]; then
+    if [[ "$#" -le 0 ]]; then
         break
     fi
 
-    lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+    lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
     case $lowerI in
-        -?|-h|--help)
+        -\?|-h|--help)
             usage
             exit 1
             ;;
@@ -111,7 +111,7 @@ while :; do
             __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
             __CodeName=buster
             __LLDB_Package="liblldb-6.0-dev"
-            __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg"
+            __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg"
             ;;
         arm64)
             __BuildArch=arm64
@@ -189,17 +189,17 @@ while :; do
             fi
             ;;
         xenial) # Ubuntu 16.04
-            if [ "$__CodeName" != "jessie" ]; then
+            if [[ "$__CodeName" != "jessie" ]]; then
                 __CodeName=xenial
             fi
             ;;
         zesty) # Ubuntu 17.04
-            if [ "$__CodeName" != "jessie" ]; then
+            if [[ "$__CodeName" != "jessie" ]]; then
                 __CodeName=zesty
             fi
             ;;
         bionic) # Ubuntu 18.04
-            if [ "$__CodeName" != "jessie" ]; then
+            if [[ "$__CodeName" != "jessie" ]]; then
                 __CodeName=bionic
             fi
             ;;
@@ -253,7 +253,7 @@ while :; do
             ;;
         --rootfsdir|-rootfsdir)
             shift
-            __RootfsDir=$1
+            __RootfsDir="$1"
             ;;
         --use-mirror)
             __UseMirror=1
@@ -266,71 +266,66 @@ while :; do
     shift
 done
 
-if [ -e "$__Keyring" ]; then
-    __Keyring="--keyring=$__Keyring"
-else
-    __Keyring=""
-fi
-
-if [ "$__BuildArch" == "armel" ]; then
+if [[ "$__BuildArch" == "armel" ]]; then
     __LLDB_Package="lldb-3.5-dev"
 fi
+
 __UbuntuPackages+=" ${__LLDB_Package:-}"
 
-if [ ! -z "$__LLVM_MajorVersion" ]; then
+if [[ -n "$__LLVM_MajorVersion" ]]; then
     __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev"
 fi
 
-if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then
-    __RootfsDir=$ROOTFS_DIR
+if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then
+    __RootfsDir="$ROOTFS_DIR"
 fi
 
-if [ -z "$__RootfsDir" ]; then
+if [[ -z "$__RootfsDir" ]]; then
     __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch"
 fi
 
-if [ -d "$__RootfsDir" ]; then
-    if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/* || true
+if [[ -d "$__RootfsDir" ]]; then
+    if [[ "$__SkipUnmount" == "0" ]]; then
+        umount "$__RootfsDir"/* || true
     fi
-    rm -rf $__RootfsDir
+    rm -rf "$__RootfsDir"
 fi
 
-mkdir -p $__RootfsDir
+mkdir -p "$__RootfsDir"
 __RootfsDir="$( cd "$__RootfsDir" && pwd )"
 
 if [[ "$__CodeName" == "alpine" ]]; then
     __ApkToolsVersion=2.9.1
-    __ApkToolsDir=$(mktemp -d)
-    wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
-    tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
-    mkdir -p $__RootfsDir/usr/bin
-    cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin
+    __ApkToolsDir="$(mktemp -d)"
+    wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir"
+    tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir"
+    mkdir -p "$__RootfsDir"/usr/bin
+    cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
 
-    $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-      -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \
-      -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \
-      -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+    "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \
+      -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \
+      -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \
+      -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \
       add $__AlpinePackages
 
-    rm -r $__ApkToolsDir
+    rm -r "$__ApkToolsDir"
 elif [[ "$__CodeName" == "freebsd" ]]; then
-    mkdir -p $__RootfsDir/usr/local/etc
+    mkdir -p "$__RootfsDir"/usr/local/etc
     JOBS="$(getconf _NPROCESSORS_ONLN)"
-    wget -O - https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
-    echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
-    echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
-    mkdir -p $__RootfsDir/tmp
+    wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+    echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
+    echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
+    mkdir -p "$__RootfsDir"/tmp
     # get and build package manager
-    wget -O -  https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz  |  tar -C $__RootfsDir/tmp -zxf -
-    cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+    wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
+    cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
     # needed for install to succeed
-    mkdir -p $__RootfsDir/host/etc
-    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install
-    rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+    mkdir -p "$__RootfsDir"/host/etc
+    ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install
+    rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
     # install packages we need.
-    INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
-    INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
+    INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
+    INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
 elif [[ "$__CodeName" == "illumos" ]]; then
     mkdir "$__RootfsDir/tmp"
     pushd "$__RootfsDir/tmp"
@@ -358,7 +353,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=http://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/${__illumosArch}/All
+    BaseUrl="$BaseUrl/packages/SmartOS/2020Q1/${__illumosArch}/All"
     echo "Downloading dependencies."
     read -ra array <<<"$__IllumosPackages"
     for package in "${array[@]}"; do
@@ -376,26 +371,26 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     wget -P "$__RootfsDir"/usr/include/net https://raw.github.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
     wget -P "$__RootfsDir"/usr/include/netpacket https://raw.github.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
     wget -P "$__RootfsDir"/usr/include/sys https://raw.github.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
-elif [[ -n $__CodeName ]]; then
-    qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo
-    cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
-    chroot $__RootfsDir apt-get update
-    chroot $__RootfsDir apt-get -f -y install
-    chroot $__RootfsDir apt-get -y install $__UbuntuPackages
-    chroot $__RootfsDir symlinks -cr /usr
-    chroot $__RootfsDir apt-get clean
+elif [[ -n "$__CodeName" ]]; then
+    qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
+    cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list"
+    chroot "$__RootfsDir" apt-get update
+    chroot "$__RootfsDir" apt-get -f -y install
+    chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages
+    chroot "$__RootfsDir" symlinks -cr /usr
+    chroot "$__RootfsDir" apt-get clean
 
-    if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/* || true
+    if [[ "$__SkipUnmount" == "0" ]]; then
+        umount "$__RootfsDir"/* || true
     fi
 
     if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
-        pushd $__RootfsDir
-        patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
+        pushd "$__RootfsDir"
+        patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch"
         popd
     fi
 elif [[ "$__Tizen" == "tizen" ]]; then
-    ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
+    ROOTFS_DIR="$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh"
 else
     echo "Unsupported target platform."
     usage;

eng/common/sdl/sdl.ps1

@@ -0,0 +1,37 @@
+
+function Install-Gdn {
+    param(
+        [string]$Path,
+
+        # If omitted, install the latest version of Guardian, otherwise install that specific version.
+        [string]$Version
+    )
+
+    $ErrorActionPreference = 'Stop'
+    Set-StrictMode -Version 2.0
+    $disableConfigureToolsetImport = $true
+    $global:LASTEXITCODE = 0
+
+    # `tools.ps1` checks $ci to perform some actions. Since the SDL
+    # scripts don't necessarily execute in the same agent that run the
+    # build.ps1/sh script this variable isn't automatically set.
+    $ci = $true
+    . $PSScriptRoot\..\tools.ps1
+
+    $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+
+    if ($Version) {
+        $argumentList += "-Version $Version"
+    }
+
+    Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
+
+    $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
+
+    if (!$gdnCliPath)
+    {
+        Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
+    }
+
+    return $gdnCliPath.FullName
+}

eng/common/templates/job/execute-sdl.yml

@@ -37,12 +37,6 @@ jobs:
   condition: eq( ${{ parameters.enable }}, 'true')
   variables:
     - group: DotNet-VSTS-Bot
-    - name: AzDOProjectName
-      value: ${{ parameters.AzDOProjectName }}
-    - name: AzDOPipelineId
-      value: ${{ parameters.AzDOPipelineId }}
-    - name: AzDOBuildId
-      value: ${{ parameters.AzDOBuildId }}
     - template: /eng/common/templates/variables/sdl-variables.yml
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}

eng/common/templates/post-build/post-build.yml

@@ -236,9 +236,9 @@ stages:
       parameters:
         enable: ${{ parameters.SDLValidationParameters.enable }}
         additionalParameters: ${{ parameters.SDLValidationParameters.params }}
-        continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+        sdlContinueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
         artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
-        downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
+        downloadArtifacts: ${{ coalesce(parameters.SDLValidationParameters.downloadArtifacts, 'false') }}
 
 - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
   - stage: publish_using_darc