#!/bin/bash -e

# Helper script to develop/debug mini-buildd.
#
# Quickstart:
#  Enter your dev chroot (preferably sid). sudo should be configured.
#  $ cd mini-buildd
#  $ ./devel installdeps
#  $ ./devel update

MBD_HOME=~mini-buildd
MBD_HTTPD="${MBD_HTTPD:-localhost:8066}"  # Use 'MBD_HTTPD=some.where.com:123 ./devel' to use something else than default localhost
MBD_TOOL="/usr/bin/mini-buildd-tool admin@${MBD_HTTPD}"  # m-b-tool shortcut for testing calls

_check_prog()
{
	local path
	for path in $(printf "${PATH}" | tr ":" " "); do
		local prog="${path}/${1}"
		if [ -x "${prog}" ]; then
			printf "I: Found: ${prog}.\n"
			return 0
		fi
	done
	printf "E: '${1}' not found in path; please install.\n" >&2
	printf "I: You may use './devel installdeps' to install all deps needed.\n" >&2
	exit 1
}

PJPATH="$(readlink -f $(dirname $0))"
PYPATH="${PJPATH}/src"
PYLINTRC="${PJPATH}/.pylintrc"
LINTIAN_CFG="${PJPATH}/.lintian.cfg"

mbd_installdeps()
{
	sudo apt-get update
	sudo apt-get --no-install-recommends install devscripts equivs

	# Debian package build dependencies; using target-release=*
	# here to always allow highest versions of any sources
	# configured (for example, for backports).
	mk-build-deps --install --root-cmd=sudo --remove --tool="apt-get --no-install-recommends --target-release='*'"

	# Extra tools needed for checks
	sudo apt-get install --no-install-recommends pycodestyle pylint3 pyflakes3 python3-apt python3-bs4 python3-keyrings.alt tidy codespell wget apache2-utils
	# Extra tools needed vc and package building
	sudo apt-get install --no-install-recommends git git-buildpackage
	# binary package dependencies so we can just dpkg -i for testing
	sudo apt-get install --no-install-recommends --target-release='*' sbuild schroot reprepro debootstrap lintian
}

# grml: The only thing I want here is that my manual test rig can do non-interactive API calls
mbd_pythonkeyringtestconfig()
{
	sudo apt-get install python3-keyrings.alt || true   # for the PlainTextKeyring (newer versions only)
	local configDir="$(python3 -c "import keyring.util.platform_; print(keyring.util.platform_.config_root())")" || true
	[ -n "${configDir}" ] || configDir="${HOME}/.local/share/python_keyring"
	mkdir -p "${configDir}"

	local configFile="${configDir}/keyringrc.cfg"
	local pkVersion=$(dpkg-query --show --showformat='${Version}' python3-keyring)
	if dpkg --compare-versions ${pkVersion} gt 7; then
		cat <<EOF >"${configFile}"
[backend]
# stretch (p-k > 8)
default-keyring=keyrings.alt.file.PlaintextKeyring
EOF
	else
		cat <<EOF >"${configFile}"
[backend]
# jessie (p-k 4.0)
default-keyring=keyring.backends.file.PlaintextKeyring
EOF
	fi
	cat "${configFile}"
}

# Find files with python code
declare -a MBD_PYFINDPARAMS=(-not -wholename './debian/*' -not -wholename './.git/*' -not -wholename './build/*' -not -wholename './.pybuild/*' -type f)
mbd_pyscripts()
{
	local f
	for f in $(find \( "${MBD_PYFINDPARAMS[@]}" \) -a -executable); do
		if head -1 "${f}" | grep --quiet "bin/python"; then
			printf "%s\n" "${f}"
		fi
	done
}

mbd_pymodules()
{
	local -a exceptions=(-true)
	[ -z "${*}" ] || exceptions=($(printf " -not -wholename %s" "${@}"))
	find -name "*.py" -a \( "${MBD_PYFINDPARAMS[@]}" \) -a \( "${exceptions[@]}" \)
}

mbd_pysources()
{
	mbd_pyscripts
	mbd_pymodules
}

mbd_pyenv()
{
	printf "export PYTHONPATH=\"${PYPATH}\"\n"
	printf "export PYLINTRC=\"${PYLINTRC}\""
}

pyenv()
{
	eval "$(mbd_pyenv)"
	python3 ./setup.py build_py
}

mbd_pylintgeneratedmembers()
{
	# Generate all identifiers with "has no xxx member" error. If
	# needed, manually add those that are _actually_
	# false-positive due to django to -> .pylintrc.
	sed -i "s/^generated-members=.*/generated-members=/" "${PYLINTRC}"  # Reset, so we actually see all these errors
	local gm=""
	for o in $(${0} pylint | grep "has no.*member" | cut -d"'" -f4 | sort | uniq); do
		gm+="${o},"
	done
	sed -i "s/^generated-members=.*/generated-members=${gm}/" "${PYLINTRC}"
	printf "${PYLINTRC} tainted.\n"
}

mbd_installdjango()
{
	dpkg -s python3-django | grep "^Version" || true
	sudo dpkg --install ../django-versions/python3-django*${1}*.deb
	dpkg -s python3-django | grep "^Version"
}

# python API has nicer support for this.
# Example call: MBD__PACKAGE="mbd-test-cpp" MBD__DIST="squeeze-test-unstable" [MBD__VERSION=1.2.3] ./devel wait4package
mbd_wait4package()
{
	local sleep=30
	printf "\nWaiting for ${MBD__PACKAGE}-${MBD__VERSION} to appear in ${MBD__DIST}:\n"
	while true; do
		# ${MBD_TOOL} show ${MBD__PACKAGE}
		if ${MBD_TOOL} show ${MBD__PACKAGE} 2>/dev/null | grep "^${MBD__DIST}\b.*${MBD__VERSION}"; then
			printf "\nOK, build\n"
			break
		else
			printf "*"
			sleep ${sleep}
		fi
	done
}

mbd_service()
{
	if sudo ischroot && [ -d /run/systemd/system ]; then
		# Seems we are in a chroot, and host is running systemd
		# The service will not be started in that case
		# For now, we really want to start the service anyway, so this is still usable in "traditional" chroots
		# (Rather use a container-based environment to test)
		sudo mv /lib/lsb/init-functions.d/40-systemd /lib/lsb/init-functions.d/40-systemd.DISABLED || true
		sudo /etc/init.d/mini-buildd ${1}
		sudo mv /lib/lsb/init-functions.d/40-systemd.DISABLED /lib/lsb/init-functions.d/40-systemd || true
	else
		sudo service mini-buildd "${1}"
	fi
}

# temporary debug helper only
# This shows bug https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=877637 (debian/rules now has a workaround).
mbd_sphinxrepro()
{
	# export PYTHONHASHSEED=random  # default, buggy
	# export PYTHONHASHSEED=0       # workaround

	local b
	for b in 1 2; do
		./debian/rules clean
		rm -rf "sphinx-${b}"
		/usr/share/sphinx/scripts/python3/sphinx-build ./doc/ ./build/sphinx/html/
		cp -a build/sphinx/ sphinx-${b}
	done
	diff -r -u -x *.pickle sphinx-1/ sphinx-2/
	printf "OK: sphinx HTML identical"
}

# Create "big files" for HTTPD tests and benchamrking.
# POST: MBD_HTTPD_TESTFILES (fileItem: sha1sum) will be available globally.
mbd_gen_httpd_testfiles()
{
	declare -g -A MBD_HTTPD_TESTFILES=()
	local fileSize relPath="repositories/test/pool/main/m/mbd-test-cpp"
	for fileSize in 5 50 100; do
		local id="file${fileSize}M"
		local of="/var/lib/mini-buildd/${relPath}/${id}.bin"
		[ -e "${of}" ] || sudo dd if="/dev/urandom" of="${of}" count="${fileSize}" bs="1M" >/dev/null 2>&1
		MBD_HTTPD_TESTFILES["${id}:http://${MBD_HTTPD}/${relPath}/${id}.bin"]="$(sha1sum < "${of}")"
	done
}

#
# Runner functions: mbd_run:<sequence>:<level>:<name>
#
# <sequence>: Two-digit sequence number (0-9) -- order in which to run.
# <level>: Higher second digit, higher "cost".
#  00-09: deploy: Actions needed to test-deploy only.
#  10-19: check : Static tests, fully automatic.
#  20-29: test  : Live tests, system tests.

mbd_run:00:10:pycodestyle()
{
	_check_prog pycodestyle
	(
		pyenv
		pycodestyle --show-source --show-pep8 --config=${PJPATH}/.pycodestyle $(mbd_pysources)
	)
}

mbd_run:00:12:pymisc()
{
	local f regex
	for f in $(mbd_pysources); do
		for regex in \
			^\#\ -\\*-\ coding:\ utf-8\ -\\*-; do
			printf "Checking '${f}' for '${regex}'..."
			grep --quiet "${regex}" "${f}"
			printf "OK.\n"
		done
	done
}

mbd_run:00:12:codespell()
{
	local ups=$(codespell --quiet-level=2 $(mbd_pysources))
	if [ -n "${ups}" ]; then
		printf "${ups}\n" >&2
		return 1
	fi
}

mbd_run:00:12:htmlmisc()
{
	local class errors=0 ignoreRegex="mbd-action-.*"

	local templateTags=$(grep "^def [[:alnum:]]" src/mini_buildd/templatetags/mini_buildd_tags.py | cut -d" " -f2 | cut -d "(" -f1 | tr "\n" " ")
	printf "I: Found django template tags: %s\n" "${templateTags}" >&2
	local idSelectors=$(grep -h -o "#mbd[[:alnum:]\-]\+" src/mini_buildd/static/css/*.css | sort -r | uniq | cut -c 2- | tr "\n" " ")
	printf "I: Found CSS ID selectors: %s\n" "${idSelectors}" >&2

	for token in ${templateTags} ${idSelectors}; do
		if [[ ${token} =~ ${ignoreRegex} ]]; then
			printf "Skipping %s.\n" "${token}"
		else
			if ! grep -r -q "${token}" src/mini_buildd/templates/; then
				printf "\nE: Token unused: %s:\n" "${token}" >&2
				grep -r "${token}" src/
				printf "\n"
				errors+=1
			fi
		fi
	done
	printf "HTML: %s unused tokens found.\n" "${errors}"
	return ${errors}
}



mbd_run:00:12:pyflakes()
{
	_check_prog pyflakes3
	(
		pyenv
		pyflakes3 $(mbd_pysources)
	)
}

mbd_run:01:10:pylint()
{
	_check_prog pylint3
	(
		pyenv
		printf "N: pylint checking modules...\n"
		pylint3 --jobs=2 $(mbd_pymodules ./doc/conf.py)  # ./doc/conf.py can't be used pylint-conform
		printf "N: pylint checking scripts...\n"
		pylint3 --jobs=2 --disable=C0103 $(mbd_pyscripts)
		printf "W: Overall locally disabled checks: %s\n" "$(cat $(mbd_pysources) | grep --count "pylint:.*disable")"  # No good way to keep track with pylint itself
	)
}

mbd_run:01:12:pydoctests()
{
	(
		pyenv
		for m in $(mbd_pymodules ./setup.py ./doc/conf.py); do  # ./setup.py && ./doc/conf.py can't be used for doctests
			local module="$(basename $(tr '/' '.' <<< ${m:4}) '.py' | cut -d. -f2-)"
			printf "=> Doctest on %s (%s)\n" "${m}" "${module}"
			( cd ./src/ && ./run-doctest "${module}" )
		done
		python3 -m doctest -v src/mini-buildd src/mini-buildd-tool
	)
}

# Build package as fast as possible
mbd_run:02:00:changelog()
{
	# Checking changelog (must be unchanged)...
	git diff-index --exit-code HEAD debian/changelog
	trap "git checkout debian/changelog" EXIT
	gbp dch --snapshot --auto
}

# Build package as fast as possible
mbd_run:03:00:buildfast()
{
	DEB_BUILD_OPTIONS+="nodoc nocheck" debuild --no-lintian -us -uc
}

# Build package again (build must have been run) w/ checks && doc
mbd_run:03:12:buildcheck()
{
	debuild -us -uc --lintian-opts --cfg="${LINTIAN_CFG}"
}

# This also checks "full" package building (with doc and check)
mbd_run:04:21:debrepro()
{
	debrepro
}

mbd_run:10:23:remove()
{
	mbd_service stop || true
	sudo dpkg --${1:-remove} mini-buildd mini-buildd-utils mini-buildd-doc python3-mini-buildd python-mini-buildd mini-buildd-common
}

mbd_run:10:23:purge()
{
	mbd_run:10:23:remove purge

	# Also test mini-buildd's internal sbuild key workaround
	sudo rm -v -f /var/lib/sbuild/apt-keys/*
}

mbd_run:11:00:install()
{
	cat ./devel.debconf.selections | sudo debconf-set-selections --verbose -
	sudo debi --with-depends
}

mbd_run:12:00:restart()
{
	mbd_service restart
	until ${MBD_TOOL} status; do
		sleep 1
	done
}

mbd_run:13:20:apicalls()
{
	${MBD_TOOL} status
	${MBD_TOOL} getkey
	${MBD_TOOL} getdputconf
	${MBD_TOOL} getsourceslist wheezy
}

mbd_run:13:20:tidy()
{
	local url
	for url in \
		http://${MBD_HTTPD}/mini_buildd \
		http://${MBD_HTTPD}/mini_buildd/api \
		http://${MBD_HTTPD}/mini_buildd/api?command=getdputconf \
		http://${MBD_HTTPD}/accounts/login/ \
		; do
		printf "\n=> Testing HTML: ${url}\n"
		wget --output-document=- "${url}" | tidy -output /dev/null
	done
}

mbd_run:14:23:auto-setup()
{
	mbd_pythonkeyringtestconfig

	if [ "${MINI_BUILDD_DEVEL_BATCH_MODE}" != "batch-mode" ]; then
		cat <<EOF
Automatically run the whole 'Admin Quickstart', all defaults,
Debian wizards, completely non-interactive.

Just give the admin password once when asked.

This is for quick bootstrapping to test-drive it, debugging and
very hasty lads.

Don't use if for, or on an existing production setup!
This will run for quite some time!
This will automatically download (most likely from internet) tons of data!

EOF
		read -p"Ctr-C to abort, RETURN to continue" dummy
	fi

	${MBD_TOOL} autosetup

	# Build keyring packages
	${MBD_TOOL} keyringpackages

	# Build test packages
	${MBD_TOOL} testpackages

	MBD__PACKAGE="mbd-test-cpp" MBD__DIST="stretch-test-unstable" mbd_wait4package
}

mbd_run:15:23:migrate()
{
	# Migration of test packages
	${MBD_TOOL} migrate mbd-test-cpp stretch-test-unstable --confirm=migrate
	${MBD_TOOL} migrate mbd-test-cpp stretch-test-testing --confirm=migrate
}

mbd_run:16:23:extra-packages()
{
	# Extra test packages when available
	[ ! -d ../test-packages/ ] || dput --force --unchecked --no-upload-log mini-buildd-$(hostname) ../test-packages/*.changes
}

mbd_run:19:23:httpd-testfiles()
{
	mbd_gen_httpd_testfiles
	local item
	for item in ${!MBD_HTTPD_TESTFILES[@]}; do
		local id=$(cut -d: -f1 <<<${item}) url=$(cut -d: -f2- <<<${item})
		local of="$(mktemp)"
		wget --output-document="${of}" "${url}"
		local sha1sum="$(sha1sum < "${of}")"
		if [ "${sha1sum}" = "${MBD_HTTPD_TESTFILES[${item}]}" ]; then
			printf "HTTP download OK: %s=%s %s" "${id}" "${of}" "${sha1sum}"
		else
			printf "HTTP download FAILED: %s=%s %s != %s" "${id}" "${of}" "${sha1sum}" "${MBD_HTTPD_TESTFILES[${item}]}"
			return 1
		fi
	done
}

mbd_run:19:23:httpd-benchmark()
{
	mbd_gen_httpd_testfiles

	_ab_val() { cut -d: -f2- | cut -d[ -f1 | tr -d '[:space:]'; }

	local abRequests=50 item

	for item in django:http://${MBD_HTTPD}/mini_buildd/repositories/test/ index:http://${MBD_HTTPD}/repositories/test/pool/main/m/mbd-test-cpp/ ${!MBD_HTTPD_TESTFILES[@]}; do
		local id=$(cut -d: -f1 <<<${item})
		local url=$(cut -d: -f2- <<<${item})
		local c
		for c in 1 4; do
			local abResult=$(ab -n "${abRequests}" -c "${c}" "${url}")
			local server=$(grep "^Server Software:.*" <<<${abResult} | _ab_val)
			if (( c > 1 )); then
				local tpr=$(grep "^Time per request:.*(mean, across all concurrent requests)" <<<${abResult} | _ab_val)
			else
				local tpr=$(grep "^Time per request:.*(mean)" <<<${abResult} | _ab_val)
			fi
			{
				if grep "^Non-2xx responses:" <<<${abResult}; then
					printf "%s" "${abResult}"
					printf "\n\n%s\n" "E: Above ab call has non-200 responses!"
					tpr="-1.0"
				fi
			} >&2
			# Note: decimal separator in "ab" is ".", so e need to set LANG for printf
			LANG="C.UTF-8" printf "[%20s] %-12s: % 8.3f\n" "${server}" "${id} c=${c}" "${tpr}"
		done
	done
}

mbd_sequence()  # [<levelRegex>=00] [<name>] <hr>
{
	local levelRegex="${1:-00}" name="${2}" hr="${3}"
	for func in $(declare -F | cut -d" " -f3- | grep "^mbd_run:[[:alnum:]][[:alnum:]]:${levelRegex}:${name}" | sort || true); do
		if [ -n "${hr}" ]; then
			printf "%s " "$(cut -d: -f4 <<<${func})"
		else
			printf "%s " "${func}"
		fi
	done
}

mbd_run()  # [<levelRegex>=00] [<name>]
{
	local -a info=()
	local func totalStartStamp=$(date +%s)
	local -i count=0
	for func in $(mbd_sequence "${1}" "${2}"); do
		printf "I: Running %s...\n" "${func}"
		local startStamp=$(date +%s)
		${func}
		count+=1
		info+=("$(printf "OK (%03d seconds): %s" "$(($(date +%s) - startStamp))" "${func}")")
	done
	if ((count <= 0)); then
		printf "E: No runs for this sequence filter: ${1} ${2}"
		return 1
	fi
	printf "\nSequence results (%03d seconds):\n" "$(($(date +%s) - totalStartStamp))"
	printf "%s\n" "${info[@]}"
	printf "\nOK, %s runs succeeded ($(date)).\n" "${count}"
}

# Shortcuts
declare -A MBD_RUN_SHORTCUTS=(
	["check"]="10"
	["update"]="00"
	["updatecheck"]="[0-1][0-1]"
	["updatetest"]="[0-9][0-2]"
	["updatetestall"]="[0-9][0-9]"
)
# We can't iterate through the associative array in the given order later, so we at least want a sorted key list as helper
MBD_RUN_SHORTCUTS_SORTED="$(printf '%s\n' "${!MBD_RUN_SHORTCUTS[@]}" | sort -n)"

mbd_logcat()
{
	tail --follow=name --retry /var/lib/mini-buildd/var/log/daemon.log /var/lib/mini-buildd/var/log/access.log
}

main()
{
	if [ -z "${1}" ]; then
		local p="./$(basename "${0}")" b=$(tput bold) i=$(tput sitm) r=$(tput sgr0)
		cat <<EOF
Usage: ${i}${p} <shortcut-or-runner-or-special> | run <groupRegex><levelRegex>${r}

mini-buildd development helper.

${b}Sequence filter shortcuts${r}:
$(for s in ${MBD_RUN_SHORTCUTS_SORTED}; do printf "  ${i}${p} %-15s${r}: (%-10s) %s\n" "${s}" "${MBD_RUN_SHORTCUTS[${s}]}" "$(mbd_sequence "${MBD_RUN_SHORTCUTS[${s}]}" "" "hr")"; done)

${b}Runners${r}:
 ${i}${p} $(mbd_sequence ".." "" "hr" | tr " " "|")

${b}Special (non-runner) targets${r}:
 ${i}${p} logcat${r}: Follow all logs (daemon and access).
 ...-> Check source for other possible esoteric calls.
EOF
	else
		local f="mbd_${1}"                    # function
		local s="${MBD_RUN_SHORTCUTS[${1}]}"  # shortcut
		if [ "$(type -t "${f}")" = "function" ]; then
			${f} "${@:2}"
		elif [ -n "${s}" ]; then
			mbd_run "${s}"
		else
			mbd_run ".." "${1}"
		fi
	fi
}

main "${@}"
