Add HTTPS support (LP: #1135163).

This is enabled by default here as it isn't unreasonably large, but won't be
usable unless GNU wget has been added to the initrd.

.gitignore

@@ -2,4 +2,4 @@ build-stamp
 choose-mirror
 mirrors_ftp.h
 mirrors_http.h
-
+mirrors_https.h

Makefile

@@ -1,4 +1,5 @@
 export USE_HTTP=1
+export USE_HTTPS=1
 #export USE_FTP=1
 USE_FTP_MANUAL=1
 
@@ -11,6 +12,10 @@ ifeq (1,${USE_HTTP})
 CFLAGS:=$(CFLAGS) -DWITH_HTTP
 templates:=$(templates) debian/choose-mirror-bin.templates.http-in
 endif
+ifeq (1,${USE_HTTPS})
+CFLAGS:=$(CFLAGS) -DWITH_HTTPS
+templates:=$(templates) debian/choose-mirror-bin.templates.https-in
+endif
 ifeq (1,${USE_FTP})
 CFLAGS:=$(CFLAGS) -DWITH_FTP
 templates:=$(templates) debian/choose-mirror-bin.templates.ftp.base-in
@@ -75,10 +80,13 @@ debian/iso_3166.tab:
 debian/httplist-countries: $(MASTERLIST) debian/iso_3166.tab
 	./mirrorlist httplist $^
 
+debian/httpslist-countries: $(MASTERLIST) debian/iso_3166.tab
+	./mirrorlist httpslist $^
+
 debian/ftplist-countries: $(MASTERLIST) debian/iso_3166.tab
 	./mirrorlist ftplist $^
 
-debian/choose-mirror-bin.templates: $(MASTERLIST) $(templates) debian/httplist-countries debian/ftplist-countries debian/iso_3166.tab
+debian/choose-mirror-bin.templates: $(MASTERLIST) $(templates) debian/httplist-countries debian/httpslist-countries debian/ftplist-countries debian/iso_3166.tab
 	# Grab ISO codes from iso-codes package
 	./get-iso-codes
 	# Build the templates
@@ -87,10 +95,13 @@ debian/choose-mirror-bin.templates: $(MASTERLIST) $(templates) debian/httplist-c
 mirrors_http.h: $(MASTERLIST) debian/iso_3166.tab
 	if [ "$$USE_HTTP" ]; then ./mirrorlist http $^; fi
 
+mirrors_https.h: $(MASTERLIST) debian/iso_3166.tab
+	if [ "$$USE_HTTPS" ]; then ./mirrorlist https $^; fi
+
 mirrors_ftp.h: $(MASTERLIST) debian/iso_3166.tab
 	if [ "$$USE_FTP" ]; then ./mirrorlist ftp $^; fi
 
-choose-mirror.o: mirrors_http.h mirrors_ftp.h
+choose-mirror.o: mirrors_http.h mirrors_https.h mirrors_ftp.h
 
 $(BIN): $(OBJS)
 	$(CC) -o $(BIN) $(OBJS) $(LIBS) $(LDFLAGS)
@@ -111,9 +122,13 @@ http: CFLAGS:=-Os -Wall -g -D_GNU_SOURCE -DWITH_HTTP -DSMALL
 http: clean strip
 	ls -l $(BIN)
 
+https: CFLAGS:=-Os -Wall -g -D_GNU_SOURCE -DWITH_HTTPS -DSMALL
+https: clean strip
+	ls -l $(BIN)
+
 clean:
 	rm -f $(BIN) $(OBJS) *~ mirrors_*.h
-	rm -f debian/templates-countries debian/httplist-countries debian/ftplist-countries
+	rm -f debian/templates-countries debian/httplist-countries debian/httpslist-countries debian/ftplist-countries
 	rm -f demo demo.templates
 	rm -rf debian/iso-codes/ debian/pobuild*/
 	rm -f debian/iso_3166.tab

choose-mirror.c

@@ -10,6 +10,9 @@
 #ifdef WITH_HTTP
 #include "mirrors_http.h"
 #endif
+#ifdef WITH_HTTPS
+#include "mirrors_https.h"
+#endif
 #ifdef WITH_FTP
 #include "mirrors_ftp.h"
 #endif
@@ -21,6 +24,8 @@
 #error Only one of WITH_FTP and WITH_FTP_MANUAL can be defined
 #endif
 
+#define MAX_PROTOCOLS 3
+
 static struct debconfclient *debconf;
 static char *protocol = NULL;
 static char *country  = NULL;
@@ -84,9 +89,9 @@ static char *debconf_list(char *list[]) {
 }
 
 /*
- * Returns the correct mirror list, depending on whether protocol is
- * set to http or ftp. Do NOT free the structure - it is a pointer to
- * the static list in mirrors_protocol.h
+ * Returns the correct mirror list, depending on the value of "protocol".
+ * Do NOT free the structure - it is a pointer to the static list in
+ * mirrors_protocol.h.
  */
 static struct mirror_t *mirror_list(void) {
 	assert(protocol != NULL);
@@ -95,6 +100,10 @@ static struct mirror_t *mirror_list(void) {
 	if (strcasecmp(protocol, "http") == 0)
 		return mirrors_http;
 #endif
+#ifdef WITH_HTTPS
+	if (strcasecmp(protocol, "https") == 0)
+		return mirrors_https;
+#endif
 #ifdef WITH_FTP
 	if (strcasecmp(protocol, "ftp") == 0)
 		return mirrors_ftp;
@@ -243,13 +252,54 @@ static int cross_validate_release(struct release_t *release) {
 	return ret;
 }
 
+int cached_wget_is_gnu = -1;
+
+static int wget_is_gnu(void) {
+	if (cached_wget_is_gnu == -1)
+		cached_wget_is_gnu = (system("wget --version 2>/dev/null | grep -q 'GNU Wget'") == 0) ? 1 : 0;
+	return cached_wget_is_gnu;
+}
+
+char *append(char *str, const char *suffix) {
+	size_t len, newlen;
+
+	len = strlen (str);
+	newlen = len + strlen(suffix);
+	str = di_realloc(str, newlen + 1);
+	strcpy(str + len, suffix);
+	return str;
+}
+
+char *get_wget_options(void) {
+	char *options;
+
+	if (wget_is_gnu()) {
+		options = strdup("--no-verbose");
+		if (strcasecmp(protocol, "https") == 0) {
+			debconf_get(debconf, "debian-installer/allow_unauthenticated_ssl");
+			if (strcmp(debconf->value, "true") == 0)
+				options = append(options, " --no-check-certificate");
+		}
+	} else {
+		if (strcasecmp(protocol, "https") == 0)
+			/* We shouldn't normally get here, but let's make it
+			 * easier to debug in case somebody has hit us over
+			 * the head with preseeding.
+			 */
+			fputs("busybox wget does not support https\n", stderr);
+		options = strdup("-q");
+	}
+
+	return options;
+}
+
 /*
  * Fetch a Release file, extract its Suite and Codename and check its valitity.
  */
 static int get_release(struct release_t *release, const char *name) {
 	char *command;
 	FILE *f = NULL;
-	char *hostname, *directory;
+	char *wget_options, *hostname, *directory;
 	char line[80];
 	char *p;
 	char buf[SUITE_LENGTH];
@@ -270,11 +320,13 @@ static int get_release(struct release_t *release, const char *name) {
 		p[0] = '\0';
 	}
 
-	command = xasprintf("wget -q %s://%s%s/dists/%s/Release -O - | grep -E '^(Suite|Codename):'",
-			    protocol, hostname, directory, name);
+	wget_options = get_wget_options();
+	command = xasprintf("wget %s %s://%s%s/dists/%s/Release -O - | grep -E '^(Suite|Codename):'",
+			    wget_options, protocol, hostname, directory, name);
 	di_log(DI_LOG_LEVEL_DEBUG, "command: %s", command);
 	f = popen(command, "r");
 	free(command);
+	free(wget_options);
 	free(hostname);
 	free(directory);
 
@@ -460,29 +512,59 @@ static int check_base_on_cd(void) {
 	return 0;
 }
 
+static int fill_protocols(const char **protocols) {
+	int i = 0;
+
+#ifdef WITH_HTTP
+	protocols[i++] = "http";
+#endif
+#ifdef WITH_HTTPS
+	if (wget_is_gnu())
+		protocols[i++] = "https";
+#endif
+#if defined (WITH_FTP) || defined (WITH_FTP_MANUAL)
+	protocols[i++] = "ftp";
+#endif
+	protocols[i] = NULL;
+
+	return i;
+}
+
 static int choose_protocol(void) {
-#if defined (WITH_HTTP) && (defined (WITH_FTP) || defined (WITH_FTP_MANUAL))
-	/* Both are supported, so ask. */
-	debconf_subst(debconf, DEBCONF_BASE "protocol", "protocols", "http, ftp");
+	const char *protocols[MAX_PROTOCOLS + 1];
+	int i;
+	size_t len;
+	char *choices;
+
+	if (fill_protocols(protocols) <= 1)
+		return 0;
+
+	/* More than one protocol is supported, so ask. */
+	len = strlen(protocols[0]);
+	for (i = 1; protocols[i]; i++)
+		len += 2 + strlen(protocols[i]);
+	choices = di_malloc(len + 1);
+	strcpy(choices, protocols[0]);
+	for (i = 1; protocols[i]; i++) {
+		strcat(choices, ", ");
+		strcat(choices, protocols[i]);
+	}
+	debconf_subst(debconf, DEBCONF_BASE "protocol", "protocols", choices);
+	di_free(choices);
 	debconf_input(debconf, "medium", DEBCONF_BASE "protocol");
-#endif
 	return 0;
 }
 
 static int get_protocol(void) {
-#if defined (WITH_HTTP) && (defined (WITH_FTP) || defined (WITH_FTP_MANUAL))
-	debconf_get(debconf, DEBCONF_BASE "protocol");
-	protocol = strdup(debconf->value);
-#else
-#ifdef WITH_HTTP
-	debconf_set(debconf, DEBCONF_BASE "protocol", "http");
-	protocol = "http";
-#endif
-#ifdef WITH_FTP
-	debconf_set(debconf, DEBCONF_BASE "protocol", "ftp");
-	protocol = "ftp";
-#endif
-#endif /* WITH_HTTP && WITH_FTP */
+	const char *protocols[MAX_PROTOCOLS + 1];
+
+	if (fill_protocols(protocols) > 1) {
+		debconf_get(debconf, DEBCONF_BASE "protocol");
+		protocol = strdup(debconf->value);
+	} else {
+		debconf_set(debconf, DEBCONF_BASE "protocol", protocols[0]);
+		protocol = strdup(protocols[0]);
+	}
 	return 0;
 }
 
@@ -773,7 +855,7 @@ int set_codename (void) {
 int check_arch (void) {
 	char *command;
 	FILE *f = NULL;
-	char *hostname, *directory, *codename = NULL;
+	char *wget_options, *hostname, *directory, *codename = NULL;
 	int valid = 0;
 
 	hostname = add_protocol("hostname");
@@ -790,11 +872,13 @@ int check_arch (void) {
 	if (strlen(debconf->value) > 0) {
 		codename = strdup(debconf->value);
 
-		command = xasprintf("wget -q %s://%s%s/dists/%s/main/binary-%s/Release -O - | grep ^Architecture:",
-				    protocol, hostname, directory, codename, ARCH_TEXT);
+		wget_options = get_wget_options();
+		command = xasprintf("wget %s %s://%s%s/dists/%s/main/binary-%s/Release -O - | grep ^Architecture:",
+				    wget_options, protocol, hostname, directory, codename, ARCH_TEXT);
 		di_log(DI_LOG_LEVEL_DEBUG, "command: %s", command);
 		f = popen(command, "r");
 		free(command);
+		free(wget_options);
 
 		if (f != NULL) {
 			char buf[SUITE_LENGTH];
@@ -851,6 +935,15 @@ int main (int argc, char **argv) {
 
 	di_system_init("choose-mirror");
 
+#ifdef WITH_HTTPS
+	debconf_register(debconf, "mirror/http/mirror", "mirror/https/mirror");
+	debconf_register(debconf,
+			 "mirror/http/hostname", "mirror/https/hostname");
+	debconf_register(debconf,
+			 "mirror/http/directory", "mirror/https/directory");
+	debconf_register(debconf, "mirror/http/proxy", "mirror/https/proxy");
+#endif
+
 	while (state >= 0 && states[state]) {
 		int res;
 

debian/changelog

@@ -1,3 +1,11 @@
+choose-mirror (2.55) UNRELEASED; urgency=medium
+
+  * Add HTTPS support (LP: #1135163).  This is enabled by default here as it
+    isn't unreasonably large, but won't be usable unless GNU wget has been
+    added to the initrd.
+
+ -- Colin Watson <cjwatson@debian.org>  Tue, 11 Feb 2014 13:42:20 +0000
+
 choose-mirror (2.54) unstable; urgency=low
 
   * Update Mirrors.masterlist

debian/choose-mirror-bin.templates.https-in

@@ -0,0 +1,20 @@
+# Templates only needed by choose-mirror if https support is enabled.
+# Most of the mirror/https/* questions are registered dynamically from
+# mirror/http/*.
+# For now, this file is intentionally left untranslated to avoid noise in
+# .po files; all its template text is shared with
+# debian/choose-mirror-bin.templates.http-in.
+
+Template: mirror/https/countries
+Type: select
+# :sl1:
+Choices-C: manual
+__Choices: enter information manually
+# Default hardcoded to US for now.  If the use of HTTPS becomes popular then
+# we may want to do the same kind of language-specific default as we do in
+# debian/choose-mirror-bin.templates.http-in.
+Default: US
+# :sl1:
+_Description: Debian archive mirror country:
+ The goal is to find a mirror of the Debian archive that is close to you on the network -- be
+ aware that nearby countries, or even your own, may not be the best choice.

mirrorlist

@@ -3,9 +3,9 @@
 # Note that there will be duplicate strings in the generated file.
 # I am relying on the c compiler to fix this, which gcc does.
 #
-# Pass in the type of mirror we are interested in (http or ftp), 
-# or use httplist or ftplist to generate a list of country codes for the
-# mirror type.
+# Pass in the type of mirror we are interested in (http, https, or ftp), or
+# use httplist, httpslist, or ftplist to generate a list of country codes
+# for the mirror type.
 use strict;
 
 my $type = shift || die "please specify mirror type\n";
@@ -30,8 +30,6 @@ close ISO3166TAB;
 # Slurp in the mirror file.
 my @data;
 my %countries;
-my %http_countries;
-my %ftp_countries;
 my $id=-1;	# incremented to 0 when first site is seen
 open (IN, $input) or die "$input: $!";
 while (<IN>) {

mirrors.h

@@ -1,5 +1,5 @@
 /*
- * Data structure for representing http and ftp mirror information.
+ * Data structure for representing http, https, and ftp mirror information.
  * Contains essentially the same information as Mirrors.masterlist,
  * but only as much information as is necessary.
  */
@@ -13,7 +13,7 @@ struct mirror_t {
 
 /*
  * The string defined below must match the string used in the templates
- * (http and ftp) for this option.
+ * (http, https, and ftp) for this option.
  */
 #define MANUAL_ENTRY "manual"
 

mktemplates

@@ -29,6 +29,11 @@ HTTPCODECHOICES="$(cut -f1 debian/httplist-countries | xargs | sed 's/ /, /g')"
 HTTPCHOICES="$(country_names debian/httplist-countries)"
 printf " Done.\n"
 
+printf "Creating the list of countries for HTTPS mirrors..."
+HTTPSCODECHOICES="$(cut -f1 debian/httpslist-countries | xargs | sed 's/ /, /g')"
+HTTPSCHOICES="$(country_names debian/httpslist-countries)"
+printf " Done.\n"
+
 printf "Creating the list of countries for FTP mirrors..."
 FTPCODECHOICES="$(cut -f1 debian/ftplist-countries | xargs | sed 's/ /, /g')"
 FTPCHOICES="$(country_names debian/ftplist-countries)"
@@ -42,6 +47,7 @@ printf "Insert the lists of choices into the templates file..."
 	done
 ) | debian/templates-build.pl "$DEB_HOST_ARCH" | \
     perl -pe 'if (m,http/countries$,) { $found = 2; } elsif ($found and /^Choices-C:/ && length "'"$HTTPCODECHOICES"'") { s/$/, '"$HTTPCODECHOICES"'/; $found -= 1; }  elsif ($found and /^__Choices:/ && length "'"$HTTPCHOICES"'") { s/$/'"$HTTPCHOICES"'/; $found -= 1; }' | \
+    perl -pe 'if (m,https/countries$,) { $found = 2; } elsif ($found and /^Choices-C:/ && length "'"$HTTPSCODECHOICES"'") { s/$/, '"$HTTPSCODECHOICES"'/; $found -= 1; }  elsif ($found and /^__Choices:/ && length "'"$HTTPSCHOICES"'") { s/$/'"$HTTPSCHOICES"'/; $found -= 1; }' | \
     perl -pe 'if (m,ftp/countries$,) { $found = 2; } elsif ($found and /^Choices-C:/ && length "'"$FTPCODECHOICES"'") { s/$/, '"$FTPCODECHOICES"'/;  $found -= 1; } elsif ($found and /^__Choices:/ && length "'"$FTPCHOICES"'") { s/$/'"$FTPCHOICES"'/;  $found -= 1; }' \
     >debian/templates.tmp
 printf " Done.\n"