Merge pull request #269 from pmw/sb

CA-148495: secure build.

mk/archive-push.sh

@@ -30,7 +30,7 @@
 
 set -eu
 
-DISABLE_PUSH=1
+# DISABLE_PUSH=1
 source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/declarations.sh"
 
 if [ ${XS_BRANCH} = "trunk" ]
@@ -44,8 +44,17 @@ then
   fi
 fi
 
+# Secure build: update buildtools, copy output to local disk, then to remote.
+cd ${OUTPUT_DIR}
+if [ "${BUILD_KIND:+$BUILD_KIND}" = production ]
+then
+     $STORE_FILES remoteupdate xensb.uk.xensource.com xenbuild git://hg.uk.xensource.com/closed/windows buildtools.git /usr/groups/build/windowsbuilds
+     $STORE_FILES store $SECURE_BUILD_ARCHIVE_UNC $get_JOB_NAME $BUILD_NUMBER *
+     $STORE_FILES remotestore xensb.uk.xensource.com xenbuild /usr/groups/build/windowsbuilds buildtools.git /usr/groups/build/windowsbuilds/WindowsBuilds $SECURE_BUILD_ARCHIVE_UNC $get_JOB_NAME $BUILD_NUMBER *
+fi
+
 #update local xenadmin-ref.hg repository
-cp ${OUTPUT_DIR}/{manifest,latest-successful-build} ${ROOT}/xenadmin-ref.hg
+cp ${OUTPUT_DIR}/{manifest,latest-*-build} ${ROOT}/xenadmin-ref.hg
 cd ${ROOT}/xenadmin-ref.hg && hg commit -m "Latest successful build ${get_BUILD_ID}"
 
 if [ ${XS_BRANCH} = "trunk" ]

mk/bumpBuildNumber.sh

@@ -30,15 +30,20 @@
 
 set -eu
 
-JENKINS_SERVER=http://tocco.uk.xensource.com:8080
-
-source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/declarations.sh"
-
 if [ $get_JOB_NAME = "devbuild" ] ; then
     echo Warning: devbuild detected so we will skip the build number increment. All dev builds will have build number 0.
     exit 0
 fi
 
+source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/declarations.sh"
+
+if [ "${BUILD_KIND:+$BUILD_KIND}" = production ]
+then
+    JENKINS_SERVER=http://tizon-1.xs.cbg.ccsi.eng.citrite.net:8080
+else
+    JENKINS_SERVER=http://tocco.uk.xensource.com:8080
+fi
+
 url="${JENKINS_SERVER}/job/${get_JOB_NAME}/"
 if curl -s --head --fail "${url}"; then
   echo "URL exists: ${url}"
@@ -47,25 +52,10 @@ else
   exit 1
 fi
 
-PSQL="ssh -q xenbuild@xenbuilder.uk.xensource.com PGPASSWORD=xenadmindb psql -q -A -t xenbuilder xenadmin"
+NEXT_BN=$(curl "http://hg.uk.xensource.com/cgi/next-xenadmin?job=$get_JOB_NAME&number=$get_BUILD_NUMBER&rev=$get_REVISION")
 
-QUERY="""INSERT INTO xenadmin_builds (build_number,job,revision) SELECT ${get_BUILD_NUMBER},'${get_JOB_NAME}','${get_REVISION}' WHERE NOT EXISTS ( SELECT 1 FROM xenadmin_builds WHERE build_number = ${get_BUILD_NUMBER});
-UPDATE xenadmin_builds SET job='${get_JOB_NAME}',revision='${get_REVISION}' WHERE build_number=${get_BUILD_NUMBER};
-SELECT MAX(build_number) FROM xenadmin_builds;"""
-
-echo "${QUERY}"
-
-MAX_BN=`${PSQL} << eof
-${QUERY}
-eof`
-
-echo MAX_BN=${MAX_BN}
-NEXT_BN=$(expr ${MAX_BN} + 1)
-
-echo NEXT_NB=${NEXT_BN}
-
-$PSQL -c "\"INSERT INTO xenadmin_builds (build_number,job,revision) VALUES (${NEXT_BN},'${get_JOB_NAME}','tba');\""
+echo NEXT_BN=${NEXT_BN}
 
 curl --data "nextBuildNumber=${NEXT_BN}" --header "Content-Type: application/x-www-form-urlencoded" ${JENKINS_SERVER}/job/${get_JOB_NAME}/nextbuildnumber/submit
 
-set +u
+set +u

mk/declarations.sh

@@ -135,22 +135,33 @@ SCRATCH_DIR=${ROOT}/scratch
 OUTPUT_DIR=${ROOT}/output
 TEST_DIR=/cygdrive/c/cygwin/tmp
 BUILD_ARCHIVE=${ROOT}/../builds/${get_BUILD_ID}/archive
+SECURE_BUILD_ARCHIVE_UNC=//10.80.13.10/distfiles/distfiles/windowsbuilds/WindowsBuilds/$get_JOB_NAME/$BUILD_NUMBER/
 #XENCENTER_LOGDIR="/cygdrive/c/Users/Administrator/AppData/Roaming/Citrix/XenCenter/logs"
 XENCENTER_LOGDIR="/cygdrive/c/Citrix/XenCenter/logs"
 
-#this is where the libraries stored in /usr/groups/linux/distfiles are exposed
-WEB_LIB="http://files.uk.xensource.com/linux/distfiles/windows-build"
-
-#this is where the current build will retrieve some of its dependendencies,
+# WEB_LIB is where the libraries stored in /usr/groups/linux/distfiles are exposed
+#WEB_LATEST_BUILD is where the current build will retrieve some of its dependendencies,
 #i.e. XenCenterOvf, version number, branding info and XenServer.NET;
 #use xe-phase-2-latest to ensure we use a build where phases 1 and 2 have succeeded
-WEB_LATEST_BUILD="http://www.uk.xensource.com/carbon/${XS_BRANCH}/xe-phase-2-latest"
+if [ "${BUILD_KIND:+$BUILD_KIND}" = production ]
+then
+    WEB_LIB="http://admin/linux/distfiles/windows-build"
+    WEB_LATEST_BUILD="http://admin/builds/carbon/${XS_BRANCH}/xe-phase-2-latest"
+else
+    WEB_LIB="http://files.uk.xensource.com/linux/distfiles/windows-build"
+    WEB_LATEST_BUILD="http://www.uk.xensource.com/carbon/${XS_BRANCH}/xe-phase-2-latest"
+fi
 WEB_XE_PHASE_1=${WEB_LATEST_BUILD}/xe-phase-1
 WEB_XE_PHASE_2=${WEB_LATEST_BUILD}/xe-phase-2
 
 #this is where the build will find stuff from the latest dotnet-packages build
 WEB_DOTNET="http://localhost:8080/job/carbon_${XS_BRANCH}_dotnet-packages/lastSuccessfulBuild/artifact"
 
+# used to copy results out of the secure build enclave
+BUILD_TOOLS_REPO=git://admin/git/closed/windows/buildtools.git
+BUILD_TOOLS=${SCRATCH_DIR}/buildtools.git
+STORE_FILES=${BUILD_TOOLS}/scripts/storefiles.py
+
 #check there are xenserver builds on this branch before proceeding
 wget -N -q --spider ${WEB_XE_PHASE_1}/globals || { echo 'FATAL: Unable to locate globals, xenadmin cannot be built if there is no succesfull build of xenserver published for the same branch.' ; exit 1; }
 

mk/xenadmin-build.sh

@@ -48,6 +48,12 @@ mkdir_clean ${OUTPUT_DIR}
 mkdir_clean ${BUILD_ARCHIVE}
 rm -rf ${TEST_DIR}/* ${XENCENTER_LOGDIR}/XenCenter.log || true
 
+if [ "${BUILD_KIND:+$BUILD_KIND}" = production ]
+then
+    git clone ${BUILD_TOOLS_REPO} ${BUILD_TOOLS}
+    chmod +x ${BUILD_TOOLS}/scripts/storefiles.py
+fi
+
 #the local revision numbers are the same as the local revision numbers on the remote repository;
 #also we know that xenadmin.git is not a patch queue style repository
 CSET_NUMBER=$(cd ${REPO} && git rev-list HEAD -1 && echo "")
@@ -415,7 +421,12 @@ cat ${SCRATCH_DIR}/xe-phase-1-manifest | grep xencenter-ovf >> ${OUTPUT_DIR}/man
 cat ${SCRATCH_DIR}/xe-phase-1-manifest | grep chroot-lenny >> ${OUTPUT_DIR}/manifest
 cat ${SCRATCH_DIR}/xe-phase-1-manifest | grep branding >> ${OUTPUT_DIR}/manifest
 cat ${SCRATCH_DIR}/dotnet-packages-manifest >> ${OUTPUT_DIR}/manifest
-echo ${get_BUILD_URL} >> ${OUTPUT_DIR}/latest-successful-build
+if [ "${BUILD_KIND:+$BUILD_KIND}" = production ]
+then
+    echo ${get_BUILD_URL} >> ${OUTPUT_DIR}/latest-secure-build
+else
+    echo ${get_BUILD_URL} >> ${OUTPUT_DIR}/latest-successful-build
+fi
 
 echo "Build phase succeeded at "
 date