path: root/email/aiaiai-email-lda

#!/bin/sh -efu

# Copyright 2011-2012 Intel Corporation
# Author: Artem Bityutskiy
# License: GPLv2

srcdir="$(readlink -ev -- ${0%/*}/..)"
PATH="$srcdir:$srcdir/email:$srcdir/external/libshell:$PATH"

. shell-error
. shell-args
. shell-signal
. aiaiai-sh-functions
. aiaiai-email-sh-functions

PROG="${0##*/}"
message_time="yes"

# This is a small trick to make sure the script is portable - check if 'dash'
# is present, and if yes - use it.
if can_switch_to_dash; then
	exec dash -euf -- "$srcdir/$PROG" "$@"
	exit $?
fi

show_usage()
{
	cat <<-EOF
Usage: $PROG [options] <workdir> <config.ini>

This is a very simple local LDA which filters out all e-mails except patches
and stores the result in the "<workdir>/queue and <workdir>/queue_saved"
directories (both stand-alone patches and patch-sets are stored as one mbox
file). The "<workdir>/lda_tmp" directory is used for temporary storing not
yet complete patch series.

Additionally, this program stores all the incoming mail in the
"<workdir>/mail" mail archive.

The "<config.ini>" file is the same file which "aiaiai-email-test-patchest"
uses. When this program receives an invalid e-mail (not a patch, invalid
subject, etc), it sends an e-mail notifications back. The "<config.ini>" file
is required for extracting basic information for sending the notifications (own
e-mail address, etc).

<workdir>    - the work directory
<config.ini> - the configuration file

Options:
      --reap-archive=MIN     remove all files created earlier than MIN
                             minutes ago from the mail archive directory;
      --reap-incomplete=MIN  remove all incomplete patch series older than
                             current time minus MIN minutes;
      --test-mode            test mode - work as usually, but do not send
                             replies;
  -v, --verbose              be verbose;
  -h, --help                 show this text and exit.
EOF
}

fail_usage()
{
	[ -z "$1" ] || printf "%s\n" "$1"
	show_usage
	exit 1
}

reject()
{
	local file="$1"; shift
	local reason="$1" shift
	local subj from id

	fetch_header_or_die subj "Subject" < "$file"
	fetch_header_or_die from "From" < "$file"
	fetch_header_or_die id "Message-Id" < "$file"

	verbose "Rejecting \"$from: $subj (Message-Id: $id)\""
	verbose "Reason: $1"
	rm $verbose -- "$file" >&2
}

reject_and_reply()
{
	local file="$1"; shift
	local reply_to reply_cc reply_id reply_subj to cc

	fetch_header_or_die reply_to "From" < "$file"
	fetch_header_or_die reply_subj "Subject" < "$file"
	fetch_header_or_die reply_id "Message-Id" < "$file"

	to="$(fetch_header "To" < "$mbox")"
	cc="$(fetch_header "Cc" < "$mbox")"

	reply_cc="$(merge_addresses "$to" "$cc")"

	prj="$(fetch_project_name "$reply_cc" "$cfg_ownmail")"
	verbose "Project \"$prj\""
	parse_prj_config "$cfgfile" "$prj"

	if [ -n "$pcfg_name" ] && [ "$pcfg_reply_to_all" = "1" ]; then
		# Strip own address
		reply_cc="$(merge_addresses "$reply_cc" "$pcfg_always_cc")"
		reply_cc=$(strip_address "$reply_cc" "$cfg_ownmail")
	else
		reply_cc=
	fi

	compose_email "$reply_to" "$reply_cc" "$reply_subj" "$reply_id" \
		> "$lda_tmp/mail"

	[ -z "$verbose" ] || cat -- "$lda_tmp/mail" >&2

	if [ -z "$test_mode" ]; then
		mutt -x -H "$lda_tmp/mail" </dev/null
	else
		verbose "Do not actually send the email - we are in test mode"
	fi
}

# Generate unique name for a file or directory in the "$1" subdirectory.
# Currently we use the same format: <date>_<suffix>-<counter>, where <date> is
# the current date (taken from the global $date variable), <suffix> is the
# a suffix supplied by the caller via "$2", and <counter> is usually 0, but if
# there is already a file with such name, it gets increased to 1, and so on.
generate_file_name()
{
	local where="$1"; shift
	local suffix="$1"; shift
	local path="$where/${date}_${suffix}-"
	local i=0

	while [ -e "${path}${i}" ]; do
		i="$(($i+1))"
	done

	printf "%s" "${path}${i}"
}

queue_mboxfile()
{
	local fname="$(generate_file_name "$queue" "$n")"
	cp $verbose -- "$mbox" "$queue_saved/${fname##*/}" >&2
	mv $verbose -- "$mbox" "$fname" >&2
}

find_relatives()
{
	local where="$1"; shift
	local id="$1"; shift
	local parent_id="$1"; shift

	# Find the parent, children, and brothers
	LC_ALL=C grep -i -l -r -- "In-Reply-To: $id" "$where" ||:
	if [ -n "$parent_id" ]; then
		LC_ALL=C grep -i -l -r -- "Message-Id: $parent_id" "$where" ||:
		LC_ALL=C grep -i -l -r -- "In-Reply-To: $parent_id" "$where" ||:
	fi
}

series_is_complete()
{
	local dir="$1"; shift
	local n="$1" ; shift

	# First check if we have all the non-cover patches yet
	local cnt="$(ls -1 --ignore=0 -- "$dir" | wc -l)"
	if [ "$cnt" -eq "$n" ]; then
		local first_parent="$(fetch_header "In-Reply-To" < "$dir/1")"
		if [ -n "$first_parent" ]; then
			# The first patch has a parent, must be the cover letter
			if [ -f "$dir/0" ]; then
				verbose "Series in $dir is complete and has cover letter"
				return 0
			else
				verbose "Series in $dir is not complete, we are missing the cover letter"
				return 1
			fi
		else
			verbose "Series is in $dir complete no cover letter was sent"
			return 0
		fi
	else
		verbose "Series in $dir is not complete, we have only $cnt out of $n patches"
		return 1
	fi
}

queue_series()
{
	local dir="$1"; shift
	local n="$1"; shift

	verbose "Patch-set at \"$dir\" is complete, queue it"
	# Don't add the 0th patch to the final mbox, as it is just the
	# cover letter and does not contain any patch
	for fname in $(ls --ignore=0 -A -- "$dir" | sort -n); do
		cat -- "$dir/$fname" >> "$mbox"
		printf "\n" >> "$mbox"
	done

	if [ -f "$dir/0" ]; then
		# Save the subject and message ID of the cover letter in the
		# final mbox in order to be able to reply to the cover letter
		# later.
		local subj="$(fetch_header "Subject" < "$dir/0")"
		subj="X-Aiaiai-Cover-Letter-Subject: $subj"

		local id="$(fetch_header "Message-Id" < "$dir/0")"
		id="X-Aiaiai-Cover-Letter-Message-Id: $id"

		# The below trick allows us to avoid creating a separate
		# temporary file: open the "$mbox" file, unlink, use the open
		# file descriptor for reading and redirect the output to the
		# new version of the "$mbox" file. We could instead use the
		# "sponge" tool, though.
		exec 3<$mbox
		rm $verbose "$mbox" >&2
		verbose "Adding \"$subj\""
		formail -s formail -I "$subj" <&3 > "$mbox"

		exec 3<$mbox
		rm $verbose "$mbox" >&2
		verbose "Adding \"$id\""
		formail -s formail -I "$id" <&3 > "$mbox"
	fi
	queue_mboxfile
	rm $verbose -rf -- "$dir" >&2
}

move_to_series()
{
	local file="$1"; shift
	local dir="$1"; shift
	local n="$1"; shift
	local subj
	local m

	fetch_header_or_die subj "Subject" < "$file"
	m="$(subject_m "$subj")"

	verbose "Moving patch (\"$subj\") the series directory"

	# Sanity check - n has to be the same
	if ! [ "$(subject_n "$subj")" -eq "$n" ]; then
		reject "$file" "We were processing patch m/$n, but found a relative $m/$n"
		return
	fi

	# File $m must not exist
	if [ -f "$dir/$m" ]; then
		reject "$file" "File \"$dir/$m\" already exists!"
	else
		mv $verbose -- "$file" "$dir/$m" >&2
	fi
}

process_series_mbox()
{
	local id="$1"; shift
	local m="$1"; shift
	local n="$1"; shift
	local fname
	local dir

	# Only patch 0/n or 1/n is allowed to have no parent
	local parent_id="$(fetch_header "In-Reply-To" < "$mbox")"
	if [ -z "$parent_id" ] && [ "$m" != 1 ] && [ "$m" != 0 ]; then
		reject_and_reply "$mbox" <<EOF
You sent a patch that does not conform to the requirements for Aiaiai's Local
Delivery Agent. This patch is part of a series as indicated by its subject
prefix.  However, it does not contain the correct "In-Reply-To:" header which
is required for Aiaiai to determine which patch series it belongs with.

The most common reason for this issue is from not using the git-send-email
utility, and using your own script which does not generate proper In-Reply-To:
header chains.

If you do use your own email commands, please add "--thread" to your
git-format-patch commands, in order to ensure that the patch series
contains proper threading.

If you do not understand this email, or believe you have received this email in
error, please contact "$cfg_adminname" <$cfg_adminmail>.
EOF
		return
	fi

	[ -z "$parent_id" ] || verbose "In-Reply-To: $parent_id"

	local staging_relatives
	local series_relatives
	staging_relatives="$(find_relatives "$staging" "$id" "$parent_id" | sort -u)"
	series_relatives="$(find_relatives "$series" "$id" "$parent_id" | sort -u)"

	verbose "Staging relatives: $staging_relatives"
	verbose "Series relatives:  $series_relatives"

	if [ -z "$staging_relatives" ] && [ -z "$series_relatives" ]; then
		# Save the file in the staging area
		fname="$(generate_file_name "$staging" "$m-of-$n")"
		verbose "No relatives found, temporarily save in staging"
		mv $verbose -- "$mbox" "$fname" >&2
		return
	fi

	verbose "Found relatives"

	if [ -z "$series_relatives" ]; then
		# The series directory does not exist yet - create it
		dir="$(generate_file_name "$series" "$n")"
		verbose "Creating the series directory \"$dir\""
		mkdir $verbose -- "$dir" >&2
	else
		dir="$(printf "%s" "$series_relatives" | sed -e "s/\(.*\)\/.*/\1/" | sort -u)"
		verbose "Found series directory \"$dir\""
		[ "$(printf "%s" "$dir" | wc -l)" -eq 0 ] ||
			die "Relatives live in different series directories!"
	fi

	# Move the relatives from the staging to the series directory
	move_to_series "$mbox" "$dir" "$n"

	local relative
	for relative in $staging_relatives; do
		move_to_series "$relative" "$dir" "$n"
	done

	# If the series is complete - queue it
	if series_is_complete "$dir" "$n"; then
		queue_series "$dir" "$n"
	fi
}

separator()
{
	if [ -n "$verbose" ]; then
		printf "\n%s"   "----------------------------------------------------" >&2
		printf "%s\n\n" "----------------------------------------------------" >&2
	fi
}

# Process one e-mail. This e-mail may be part of a patch-set, or an individual
# patch, or not a patch at all. The patches and non-patches are distinguished
# using the subject (see the 'subject_check()' function). Non-patches are
# dropped. Individual patches are queued right away. Patch-sets are stored in
# a temporary directory and queued only when all the of them are collected.
#
# The e-mail to process is passed in "$1" in form of a path to the mbox file.
process_mbox()
{
	local mbox="$1";

	separator

	# Make sure important headers are there
	grep -i -q -- "^Message-Id:[[:blank:]]" "$mbox" ||
		{ verbose "The \"Message-Id:\" header not found, ignoring"; return; }
	grep -i -q -- "^From:[[:blank:]]" "$mbox" ||
		{ verbose "The \"From:\" header not found, ignoring"; return; }
	grep -i -q -- "^To:[[:blank:]]" "$mbox" ||
		{ verbose "The \"To:\" header not found, ignoring"; return; }
	grep -i -q -- "^Subject:[[:blank:]]" "$mbox" ||
		{ verbose "The \"Subject:\" header not found, ignoring"; return; }

	local subj="$(fetch_header "Subject" < "$mbox")"
	local from="$(fetch_header "From" < "$mbox")"
	local id="$(fetch_header "Message-Id" < "$mbox")"

	verbose "Looking at: From: $from"
	verbose "            Subject: $subj"
	verbose "            Message-Id: $id"

	# Filter out e-mails which do not start with the right prefix"
	subject_check "$subj" ||
		{ reject "$mbox" "\"$prefix_format\" prefix not found"; return; }

	# If the patch prefix contains m/n, fetch m and n.
	local m n
	m="$(subject_m "$subj")"
	n="$(subject_n "$subj")"

	if [ -z "$m" ]; then
		verbose "Queuing stand-alone patch \"$subj\""
		queue_mboxfile
	else
		verbose "Processing member $m/$n of a series (\"$subj\")"
		[ "$n" -ne 0 ] || \
			{ reject "$mbox" "Prefix \"$prefix_format\" cannot have n = 0";
			  return; }
		process_series_mbox "$id" "$m" "$n"
	fi
}

reap_old()
{
	local dir="$1"; shift
	local min="$1"; shift

	verbose "Reaping files older than \"$min\" minutes in \"$dir\""
	find "$dir" -mindepth 1 -mmin +"$min" -exec rm $verbose -f -- "{}" ";" >&2
}

mbox=
lda_tmp_lock=
cleanup_handler()
{
	rm $verbose -rf -- "$mbox" >&2
	rm $verbose -rf -- "$lda_tmp_lock" >&2
}
set_cleanup_handler cleanup_handler

TEMP=`getopt -n $PROG -o v,h --long test-mode,reap-archive:,reap-incomplete:,verbose,help -- "$@"` ||
	fail_usage ""
eval set -- "$TEMP"

test_mode=
archive_min=
incomplete_min=
verbose=

while true; do
	case "$1" in
	--test-mode)
		test_mode=y
		;;
	--reap-archive)
		archive_min=$(opt_check_number "$1" "$2")
		shift
		;;
	--reap-incomplete)
		incomplete_min=$(opt_check_number "$1" "$2")
		shift
		;;
	-v|--verbose) verbose=-v
		;;
	-h|--help)
		show_usage
		exit 0
		;;
	--) shift; break
                ;;
	*) fail_usage "Unrecognized option: $1"
		;;
	esac
	shift
done

[ "$#" -eq 2 ] || die "Insufficient or too many arguments"

program_required "mutt" ""
program_required "find" ""
program_required "grep" ""
program_required "sed" ""
program_required "formail" ""

workdir=$(readlink -fv -- "$1"); shift
cfgfile=$(readlink -fv -- "$1"); shift

parse_config "$cfgfile"

lda_tmp="$workdir/lda_tmp"
mkdir -p $verbose -- "$lda_tmp" >&2

# In the staging directory we temporarily store patches which belong to a
# patch-set but, but we do not know which one yet.
staging="$lda_tmp/staging"
# The series directory stores partial patch-sets.
series="$lda_tmp/series"
# The mail directory sores all the incoming mail
mail="$workdir/mail"
# The queue directory stores patches/patch-sets queued for further processing
queue="$workdir/queue"
# It is also convenient for debugging to store a copy of complete patch-series
# separately
queue_saved="$workdir/queue_saved"

mkdir -p $verbose -- "$staging" "$series" "$mail" "$queue" "$queue_saved" >&2

# All file we create have the date in the name
date="$(date "+%Y-%m-%d_%H:%M:%S")"

# We get the input from stdin, and it may contain several mails. We then
# separate them, and process one-by-one. And the mail which is currently being
# processed is stored in this temporary file.
mbox="$(mktemp -t "$PROG.mbox.XXXX")"

# We lock the lda and queue directories when using them
lda_tmp_lock="$workdir/lda_tmp.lock"

verbose "Taking $lda_tmp_lock lock file (timeout - 10 min)"
lockfile -r 75 "$lda_tmp_lock"

prev=
first=y

# Read stdin and separate out individual mails
verbose "Saving incoming mbox in ${mail}/${date}.mbox"
while IFS= read -r line; do
	printf "%s\n" "$line" >> "${mail}/${date}.mbox"
	if printf "%s" "$line" | grep -i -q -- "^From [^@ ]\+@[^@ ]\+ .\+$" || \
	   printf "%s" "$line" | grep -i -q -- "^From [[:xdigit:]]\{40\} .\+$"; then
		if [ -z "$first" ]; then
			if [ -n "$prev" ]; then
				verbose "The last line of previous mbox is not blank"
				verbose "line: $line"
				verbose "prev: $prev"
				die "exiting"
			fi
			process_mbox "$mbox"
			truncate -s0 -- "$mbox"
		fi
	else
		printf "%s\n" "$prev" >> "$mbox"
	fi
	prev="$line"
	first=
done

printf "%s\n" "$prev" >> "$mbox"
process_mbox "$mbox"

[ -z "$incomplete_min" ] || reap_old "$lda_tmp" "$incomplete_min"
[ -z "$archive_min" ] || reap_old "$mail" "$archive_min"
[ -z "$archive_min" ] || reap_old "$queue_saved" "$archive_min"

rm $verbose -f -- "$lda_tmp_lock" >&2