lava-submit.sh

  1#!/usr/bin/env bash
  2# SPDX-License-Identifier: MIT
  3# shellcheck disable=SC2086 # we want word splitting
  4# shellcheck disable=SC1091 # paths only become valid at runtime
  5
  6# If we run in the fork (not from mesa or Marge-bot), reuse mainline kernel and rootfs, if exist.
  7_check_artifact_path() {
  8	_url="https://${1}/${2}"
  9	if curl -s -o /dev/null -I -L -f --retry 4 --retry-delay 15 "${_url}"; then
 10		echo -n "${_url}"
 11	fi
 12}
 13
 14get_path_to_artifact() {
 15	_mainline_artifact="$(_check_artifact_path ${BASE_SYSTEM_MAINLINE_HOST_PATH} ${1})"
 16	if [ -n "${_mainline_artifact}" ]; then
 17		echo -n "${_mainline_artifact}"
 18		return
 19	fi
 20	_fork_artifact="$(_check_artifact_path ${BASE_SYSTEM_FORK_HOST_PATH} ${1})"
 21	if [ -n "${_fork_artifact}" ]; then
 22		echo -n "${_fork_artifact}"
 23		return
 24	fi
 25	set +x
 26	error "Sorry, I couldn't find a viable built path for ${1} in either mainline or a fork." >&2
 27	echo "" >&2
 28	echo "If you're working on CI, this probably means that you're missing a dependency:" >&2
 29	echo "this job ran ahead of the job which was supposed to upload that artifact." >&2
 30	echo "" >&2
 31	echo "If you aren't working on CI, please ping @mesa/ci-helpers to see if we can help." >&2
 32	echo "" >&2
 33	echo "This job is going to fail, because I can't find the resources I need. Sorry." >&2
 34	set -x
 35	exit 1
 36}
 37
 38. "${SCRIPTS_DIR}/setup-test-env.sh"
 39
 40section_start prepare_rootfs "Preparing root filesystem"
 41
 42set -ex
 43
 44ROOTFS_URL="$(get_path_to_artifact lava-rootfs.tar.zst)"
 45[ $? != 1 ] || exit 1
 46
 47rm -rf results
 48mkdir -p results/job-rootfs-overlay/
 49
 50artifacts/ci-common/export-gitlab-job-env-for-dut.sh \
 51    > results/job-rootfs-overlay/set-job-env-vars.sh
 52cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/
 53cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/
 54
 55tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
 56ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
 57
 58# Prepare env vars for upload.
 59section_switch variables "Environment variables passed through to device:"
 60cat results/job-rootfs-overlay/set-job-env-vars.sh
 61
 62section_switch lava_submit "Submitting job for scheduling"
 63
 64touch results/lava.log
 65tail -f results/lava.log &
 66# Ensure that we are printing the commands that are being executed,
 67# making it easier to debug the job in case it fails.
 68set -x
 69PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 70	--farm "${FARM}" \
 71	--device-type "${DEVICE_TYPE}" \
 72	--boot-method "${BOOT_METHOD}" \
 73	--job-timeout-min $((CI_JOB_TIMEOUT/60 - 5)) \
 74	--dump-yaml \
 75	--pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
 76	--rootfs-url "${ROOTFS_URL}" \
 77	--kernel-url-prefix "https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}" \
 78	--kernel-external "${EXTERNAL_KERNEL_TAG}" \
 79	--first-stage-init artifacts/ci-common/init-stage1.sh \
 80	--dtb-filename "${DTB}" \
 81	--jwt-file "${S3_JWT_FILE}" \
 82	--kernel-image-name "${KERNEL_IMAGE_NAME}" \
 83	--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
 84	--visibility-group "${VISIBILITY_GROUP}" \
 85	--lava-tags "${LAVA_TAGS}" \
 86	--mesa-job-name "$CI_JOB_NAME" \
 87	--structured-log-file "results/lava_job_detail.json" \
 88	--ssh-client-image "${LAVA_SSH_CLIENT_IMAGE}" \
 89	--project-name "${CI_PROJECT_NAME}" \
 90	--starting-section "${CURRENT_SECTION}" \
 91	--job-submitted-at "${CI_JOB_STARTED_AT}" \
 92	- append-overlay \
 93		--name=kernel-build \
 94		--url="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/kernel-files.tar.zst" \
 95		--compression=zstd \
 96		--path="${CI_PROJECT_DIR}" \
 97		--format=tar \
 98	- append-overlay \
 99		--name=job-overlay \
100		--url="https://${JOB_ROOTFS_OVERLAY_PATH}" \
101		--compression=gz \
102		--path="/" \
103		--format=tar \
104	- submit \
105	>> results/lava.log