Protocols utilized in the original posts remain necessary…TFTP, DHCP, and HTTP. Cobbler remains our main provisioning platform. There are several references to Cobbler here. Some hacking may be required to permit these concepts to work on other provisioning platforms.

The result is the emergence of a platform utilizing tftp to load the iPXE boot code which, in turn, loads a custom FreeBSD bootonly ISO that performs a sysinstall(8) based installation. Only the first step of this process is executed over tftp, the remaining steps are performed over HTTP.

Abbreviated Process

This is a brief description of the overall process of building a FreeBSD system.

1. PXE boot client sends a broadcast DHCP request and the DHCP server responds with an IP based on the client’s MAC address.
2. DHCP client executes a lookup against the DHCP Server based on the MAC Address of the PXE Client and the following data is sent to the client upon discovery of a valid host entry:
   • IP Address
   • Netmask
   • Default Gateway
   • Hostname
   • Name Servers
   • next-server
   • filename
3. The PXE boot client configures networking based on the data returned above and initiates a tftp request for “filename” from “next-server”. In this scenario, the filename is the iPXE bootstrap program.
4. The iPXE bootloader runs, executing a DHCP request on the interface it was downloaded on (net0). iPXE sets “user-class” to “iPXE” when executing this request.
5. The DHCP server responds to the incoming iPXE request with the following and sets “filename” based on the value of “user-class”:
   • IP Address
   • Netmask
   • Default Gateway
   • Hostname
   • Name Servers
   • next-server
   • filename
6. iPXE executes an HTTP GET request for a configuration file derived from the DHCP options written to a leases file (described later). The response from the Cobbler server is based on the netboot-enabled attribute (In this scenario, netboot-enabled is true).
7. iPXE bootloader executes an HTTP GET for memdisk (necessary to boot the ISO) and bootonly.iso.zip (described later).
8. bootonly.iso.zip (described later) runs sysinstall and performs an OS installation on the host.

Generate A Release

Patch sysinstall

Until FreeBSD 8.4-RELEASE and 9.2-RELEASE (with the exception of -STABLE), sysinstall(8) lacked support for direct HTTP installs. Devin Teske and I worked closely to get this patch committed to stable/8 and stable/9 to add the functionality to releng/8.4 and releng/9.2.

Index: usr.sbin/sysinstall/dispatch.c
===================================================================
--- usr.sbin/sysinstall/dispatch.c	(revision 248310)
+++ usr.sbin/sysinstall/dispatch.c	(revision 248313)
@@ -103,6 +103,7 @@
     { "mediaSetFTPActive",	mediaSetFTPActive	},
     { "mediaSetFTPPassive",	mediaSetFTPPassive	},
     { "mediaSetHTTP",		mediaSetHTTP		},
+    { "mediaSetHTTPDirect",	mediaSetHTTPDirect	},
     { "mediaSetUFS",		mediaSetUFS		},
     { "mediaSetNFS",		mediaSetNFS		},
     { "mediaSetFTPUserPass",	mediaSetFTPUserPass	},
Index: usr.sbin/sysinstall/media.c
===================================================================
--- usr.sbin/sysinstall/media.c	(revision 248310)
+++ usr.sbin/sysinstall/media.c	(revision 248313)
@@ -52,6 +52,7 @@
 
 static Boolean got_intr = FALSE;
 static Boolean ftp_skip_resolve = FALSE;
+static Boolean http_skip_resolve = FALSE;
 
 /* timeout handler */
 static void
@@ -508,6 +509,139 @@
     mediaDevice->shutdown = dummyShutdown;
     return DITEM_SUCCESS | DITEM_LEAVE_MENU | what;
 }
+
+/*
+ * Return 0 if we successfully found and set the installation type to
+ * be an http server
+ */
+int
+mediaSetHTTPDirect(dialogMenuItem *self)
+{
+    static Device httpDevice;
+    char *cp, hbuf[MAXPATHLEN], *hostname, *dir;
+    struct addrinfo hints, *res;
+    int af;
+    size_t urllen;
+    int HttpPort;
+    static Device *networkDev = NULL;
+
+    mediaClose();
+    cp = variable_get(VAR_HTTP_PATH);
+    /* If we've been through here before ... */
+    if (networkDev && cp && msgYesNo("Re-use old HTTP site selection values?"))
+	cp = NULL;
+    if (!cp) {
+	if (!dmenuOpenSimple(&MenuMediaHTTPDirect, FALSE))
+	    return DITEM_FAILURE;
+	else
+	    cp = variable_get(VAR_HTTP_PATH);
+    }
+    if (!cp)
+	return DITEM_FAILURE;
+    else if (!strcmp(cp, "other")) {
+	variable_set2(VAR_HTTP_PATH, "http://", 0);
+	cp = variable_get_value(VAR_HTTP_PATH, "Please specify the URL of a FreeBSD distribution on a\n"
+				"remote http site.\n"
+				"A URL looks like this:  http://<hostname>/<path>", 0);
+	if (!cp || !*cp || !strcmp(cp, "http://")) {
+	    variable_unset(VAR_HTTP_PATH);
+	    return DITEM_FAILURE;
+	}
+	urllen = strlen(cp);
+	if (urllen >= sizeof(httpDevice.name)) {
+	    msgConfirm("Length of specified URL is %zu characters. Allowable maximum is %zu.",
+			urllen,sizeof(httpDevice.name)-1);
+	    variable_unset(VAR_HTTP_PATH);
+	    return DITEM_FAILURE;
+	}
+    }
+    if (strncmp("http://", cp, 7)) {
+	msgConfirm("Sorry, %s is an invalid URL!", cp);
+	variable_unset(VAR_HTTP_PATH);
+	return DITEM_FAILURE;
+    }
+    SAFE_STRCPY(httpDevice.name, cp);
+    SAFE_STRCPY(hbuf, cp + 7);
+    hostname = hbuf;
+
+    if (!networkDev || msgYesNo("You've already done the network configuration once,\n"
+				"would you like to skip over it now?") != 0) {
+	if (networkDev)
+	    DEVICE_SHUTDOWN(networkDev);
+	if (!(networkDev = tcpDeviceSelect())) {
+	    variable_unset(VAR_HTTP_PATH);
+	    return DITEM_FAILURE;
+	}
+    }
+    if (!DEVICE_INIT(networkDev)) {
+	if (isDebug())
+	    msgDebug("mediaSetHTTPDirect: Net device init failed.\n");
+	variable_unset(VAR_HTTP_PATH);
+	return DITEM_FAILURE;
+    }
+    if (*hostname == '[' && (cp = index(hostname + 1, ']')) != NULL &&
+	(*++cp == '\0' || *cp == '/' || *cp == ':')) {
+	++hostname;
+	*(cp - 1) = '\0';
+    }
+    else
+	cp = index(hostname, ':');
+    if (cp != NULL && *cp == ':') {
+	*(cp++) = '\0';
+	HttpPort = strtol(cp, 0, 0);
+    }
+    else
+	HttpPort = 80;
+    if ((dir = index(cp ? cp : hostname, '/')) != NULL)
+	*(dir++) = '\0';
+    if (isDebug()) {
+	msgDebug("hostname = `%s'\n", hostname);
+	msgDebug("dir = `%s'\n", dir ? dir : "/");
+	msgDebug("port # = `%d'\n", HttpPort);
+    }
+    if (!http_skip_resolve && variable_get(VAR_NAMESERVER)) {
+	msgNotify("Looking up host %s.", hostname);
+    	if (isDebug())
+	    msgDebug("Starting DNS.\n");
+	kickstart_dns();
+    	if (isDebug())
+	    msgDebug("Looking up hostname, %s, using getaddrinfo(AI_NUMERICHOST).\n", hostname);
+	af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = af;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;
+	if (getaddrinfo(hostname, NULL, &hints, &res) != 0) {
+	    if (isDebug())
+		msgDebug("Looking up hostname, %s, using getaddrinfo().\n",
+			 hostname);
+	    hints.ai_flags = AI_PASSIVE;
+	    if (getaddrinfo(hostname, NULL, &hints, &res) != 0) {
+		msgConfirm("Cannot resolve hostname `%s'!  Are you sure that"
+			" your\nname server, gateway and network interface are"
+			" correctly configured?", hostname);
+		if (networkDev)
+		    DEVICE_SHUTDOWN(networkDev);
+		networkDev = NULL;
+		variable_unset(VAR_HTTP_PATH);
+		return DITEM_FAILURE;
+	    }
+	}
+	freeaddrinfo(res);
+	if (isDebug())
+	    msgDebug("Found DNS entry for %s successfully..\n", hostname);
+    }
+    variable_set2(VAR_HTTP_HOST, hostname, 0);
+    variable_set2(VAR_HTTP_DIR, dir ? dir : "/", 0);
+    variable_set2(VAR_HTTP_PORT, itoa(HttpPort), 0);
+    httpDevice.type = DEVICE_TYPE_HTTP_DIRECT;
+    httpDevice.init = mediaInitHTTPDirect;
+    httpDevice.get = mediaGetHTTPDirect;
+    httpDevice.shutdown = dummyShutdown;
+    httpDevice.private = networkDev;
+    mediaDevice = &httpDevice;
+    return DITEM_SUCCESS | DITEM_LEAVE_MENU | DITEM_RESTORE;
+}
    
 
 int
Index: usr.sbin/sysinstall/http.c
===================================================================
--- usr.sbin/sysinstall/http.c	(revision 248310)
+++ usr.sbin/sysinstall/http.c	(revision 248313)
@@ -36,18 +36,9 @@
 
 extern const char *ftp_dirs[]; /* defined in ftp.c */
 
-static Boolean
-checkAccess(Boolean proxyCheckOnly)
+Boolean
+checkAccess(Boolean connectCheckOnly, Boolean isProxy)
 {
-/* 
- * Some proxies fetch files with certain extensions in "ascii mode" instead
- * of "binary mode" for FTP. The FTP server then translates all LF to CRLF.
- *
- * You can force Squid to use binary mode by appending ";type=i" to the URL,
- * which is what I do here. For other proxies, the LF->CRLF substitution
- * is reverted in distExtract().
- */
-
     int rv, s, af;
     bool el, found=FALSE;		    /* end of header line */
     char *cp, buf[PATH_MAX], req[BUFSIZ];
@@ -76,18 +67,26 @@
     }
     freeaddrinfo(res0);
     if (s == -1) {
-	msgConfirm("Couldn't connect to proxy %s:%s",
-		    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+	if (isProxy) {
+		msgConfirm("Couldn't connect to proxy %s:%s",
+			    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+	} else {
+		msgConfirm("Couldn't connect to server http://%s:%s/",
+			    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+	}
 	variable_unset(VAR_HTTP_HOST);
 	return FALSE;
     }
-    if (proxyCheckOnly) {
+    if (connectCheckOnly) {
        close(s);
        return TRUE;
     }
 
     msgNotify("Checking access to\n %s", variable_get(VAR_HTTP_PATH));
-    sprintf(req,"GET %s/ HTTP/1.0\r\n\r\n", variable_get(VAR_HTTP_PATH));
+    if (isProxy)
+	sprintf(req,"GET %s/ HTTP/1.0\r\n\r\n", variable_get(VAR_HTTP_PATH));
+    else
+	sprintf(req,"GET /%s/ HTTP/1.0\r\n\r\n", variable_get(VAR_HTTP_PATH));
     write(s,req,strlen(req));
 /*
  *  scan the headers of the response
@@ -108,7 +107,16 @@
 		}
 	    }
 
-	    if (!strncmp(buf,"Server: ",8)) {
+	    /* 
+	     * Some proxies fetch files with certain extensions in "ascii mode"
+	     * instead of "binary mode" for FTP. The FTP server then translates
+	     * all LF to CRLF.
+	     *
+	     * You can force Squid to use binary mode by appending ";type=i" to
+	     * the URL, which is what I do here. For other proxies, the
+	     * LF->CRLF substitution is reverted in distExtract().
+	     */
+	    if (isProxy && !strncmp(buf,"Server: ",8)) {
 		if (!strncmp(buf,"Server: Squid",13)) {
 		    variable_set2(VAR_HTTP_FTP_MODE,";type=i",0);
 		} else {
@@ -143,11 +151,11 @@
     /* 
      * First verify the proxy access
      */
-    checkAccess(TRUE);
+    checkAccess(TRUE, TRUE);
     while (variable_get(VAR_HTTP_HOST) == NULL) {
         if (DITEM_STATUS(mediaSetHTTP(NULL)) == DITEM_FAILURE)
             return FALSE;
-        checkAccess(TRUE);
+        checkAccess(TRUE, TRUE);
     }
 again:
     /* If the release is specified as "__RELEASE" or "any", then just
@@ -163,14 +171,14 @@
             sprintf(req, "%s/%s/%s", variable_get(VAR_FTP_PATH),
                 ftp_dirs[fdir], rel);
             variable_set2(VAR_HTTP_PATH, req, 0);
-            if (checkAccess(FALSE)) {
+            if (checkAccess(FALSE, TRUE)) {
                 found = TRUE;
                 break;
             }
         }
     } else {
         variable_set2(VAR_HTTP_PATH, variable_get(VAR_FTP_PATH), 0);
-        found = checkAccess(FALSE);
+        found = checkAccess(FALSE, TRUE);
     }
     if (!found) {
     	msgConfirm("No such directory: %s\n"
Index: usr.sbin/sysinstall/Makefile
===================================================================
--- usr.sbin/sysinstall/Makefile	(revision 248310)
+++ usr.sbin/sysinstall/Makefile	(revision 248313)
@@ -8,9 +8,9 @@
 MAN=	sysinstall.8
 SRCS=	anonFTP.c cdrom.c command.c config.c devices.c dhcp.c \
 	disks.c dispatch.c dist.c dmenu.c doc.c dos.c floppy.c \
-	ftp.c globals.c http.c index.c install.c installUpgrade.c keymap.c \
-	label.c main.c makedevs.c media.c menus.c misc.c modules.c \
-	mouse.c msg.c network.c nfs.c options.c package.c \
+	ftp.c globals.c http.c httpdirect.c index.c install.c \
+	installUpgrade.c keymap.c label.c main.c makedevs.c media.c menus.c \
+	misc.c modules.c mouse.c msg.c network.c nfs.c options.c package.c \
 	system.c tcpip.c termcap.c ttys.c ufs.c usb.c user.c \
 	variable.c ${_wizard} keymap.h countries.h
 
Index: usr.sbin/sysinstall/help/media.hlp
===================================================================
--- usr.sbin/sysinstall/help/media.hlp	(revision 248310)
+++ usr.sbin/sysinstall/help/media.hlp	(revision 248313)
@@ -41,6 +41,14 @@
             Options screen.
 
 
+   HTTP Direct
+            Get the distribution files directly from an HTTP server.
+
+            If you chose to enter your own URL in the HTTP Direct menu,
+            please note that all paths are *relative* to the root
+            directory of the web server.
+
+
    NFS      Get the distribution files from an NFS server somewhere
             (make sure that permissions on the server allow this!).
             If this install method hangs on you or refuses to work
Index: usr.sbin/sysinstall/sysinstall.h
===================================================================
--- usr.sbin/sysinstall/sysinstall.h	(revision 248310)
+++ usr.sbin/sysinstall/sysinstall.h	(revision 248313)
@@ -118,6 +118,7 @@
 #define VAR_FTP_STATE			"ftpState"
 #define VAR_FTP_USER			"ftpUser"
 #define VAR_FTP_HOST			"ftpHost"
+#define VAR_HTTP_DIR			"httpDirectory"
 #define VAR_HTTP_PATH			"_httpPath"
 #define VAR_HTTP_PROXY			"httpProxy"
 #define VAR_HTTP_PORT			"httpPort"
@@ -273,6 +274,7 @@
     DEVICE_TYPE_NFS,
     DEVICE_TYPE_ANY,
     DEVICE_TYPE_HTTP,
+    DEVICE_TYPE_HTTP_DIRECT,
 } DeviceType;
 
 /* CDROM mount codes */
@@ -443,6 +445,7 @@
 extern DMenu		MenuMediaDOS;		/* DOS media menu		*/
 extern DMenu		MenuMediaFloppy;	/* Floppy media menu		*/
 extern DMenu		MenuMediaFTP;		/* FTP media menu		*/
+extern DMenu		MenuMediaHTTPDirect;	/* HTTP Direct media menu	*/
 extern DMenu		MenuNetworkDevice;	/* Network device menu		*/
 extern DMenu		MenuNTP;		/* NTP time server menu		*/
 extern DMenu		MenuSecurity;		/* System security options menu*/
@@ -650,9 +653,14 @@
 extern void	mediaShutdownFTP(Device *dev);
 
 /* http.c */
+extern Boolean	checkAccess(Boolean connectCheckOnly, Boolean isProxy);
 extern Boolean	mediaInitHTTP(Device *dev);
 extern FILE	*mediaGetHTTP(Device *dev, char *file, Boolean probe);
 
+/* httpdirect.c */
+extern Boolean	mediaInitHTTPDirect(Device *dev);
+extern FILE	*mediaGetHTTPDirect(Device *dev, char *file, Boolean probe);
+
 /* globals.c */
 extern void	globalsInit(void);
 
@@ -726,6 +734,7 @@
 extern int	mediaSetFTPActive(dialogMenuItem *self);
 extern int	mediaSetFTPPassive(dialogMenuItem *self);
 extern int	mediaSetHTTP(dialogMenuItem *self);
+extern int	mediaSetHTTPDirect(dialogMenuItem *self);
 extern int	mediaSetUFS(dialogMenuItem *self);
 extern int	mediaSetNFS(dialogMenuItem *self);
 extern int	mediaSetFTPUserPass(dialogMenuItem *self);
Index: usr.sbin/sysinstall/httpdirect.c
===================================================================
--- usr.sbin/sysinstall/httpdirect.c	(revision 0)
+++ usr.sbin/sysinstall/httpdirect.c	(revision 248313)
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 1999
+ *	Philipp Mergenthaler <philipp.mergenthaler@stud.uni-karlsruhe.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "sysinstall.h"
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/param.h>
+#include <netdb.h>
+
+extern const char *ftp_dirs[]; /* defined in ftp.c */
+
+Boolean
+mediaInitHTTPDirect(Device *dev)
+{
+    bool found=FALSE;		    /* end of header line */
+    char *rel, req[BUFSIZ];
+    int fdir;
+
+    /* 
+     * First verify basic access
+     */
+    checkAccess(TRUE, FALSE);
+    while (variable_get(VAR_HTTP_HOST) == NULL) {
+        if (DITEM_STATUS(mediaSetHTTPDirect(NULL)) == DITEM_FAILURE)
+            return FALSE;
+        checkAccess(TRUE, FALSE);
+    }
+again:
+    /* If the release is specified as "__RELEASE" or "any", then just
+     * assume that the path the user gave is ok.
+     */
+    rel = variable_get(VAR_RELNAME);
+    /*
+    msgConfirm("rel: -%s-", rel);
+    */
+
+    if (strcmp(rel, "__RELEASE") && strcmp(rel, "any"))  {
+        for (fdir = 0; ftp_dirs[fdir]; fdir++) {
+            sprintf(req, "%s/%s/%s", variable_get(VAR_HTTP_DIR),
+                ftp_dirs[fdir], rel);
+            variable_set2(VAR_HTTP_PATH, req, 0);
+            if (checkAccess(FALSE, FALSE)) {
+                found = TRUE;
+                break;
+            }
+        }
+    } else {
+        variable_set2(VAR_HTTP_PATH, variable_get(VAR_HTTP_DIR), 0);
+        found = checkAccess(FALSE, FALSE);
+    }
+    if (!found) {
+    	msgConfirm("No such directory: %s\n"
+		   "please check the URL and try again.", variable_get(VAR_HTTP_PATH));
+        variable_unset(VAR_HTTP_PATH);
+        dialog_clear_norefresh();
+        clear();
+        if (DITEM_STATUS(mediaSetHTTPDirect(NULL)) != DITEM_FAILURE) goto again;
+    }
+    return found;
+}
+
+FILE *
+mediaGetHTTPDirect(Device *dev, char *file, Boolean probe)
+{
+    FILE *fp;
+    int rv, s, af;
+    bool el;			/* end of header line */
+    char *cp, buf[PATH_MAX], req[BUFSIZ];
+    struct addrinfo hints, *res, *res0;
+
+    af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = af;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_protocol = 0;
+    if ((rv = getaddrinfo(variable_get(VAR_HTTP_HOST),
+			  variable_get(VAR_HTTP_PORT), &hints, &res0)) != 0) {
+	msgConfirm("%s", gai_strerror(rv));
+	return NULL;
+    }
+    s = -1;
+    for (res = res0; res; res = res->ai_next) {
+	if ((s = socket(res->ai_family, res->ai_socktype,
+			res->ai_protocol)) < 0)
+	    continue;
+	if (connect(s, res->ai_addr, res->ai_addrlen) >= 0)
+	    break;
+	close(s);
+	s = -1;
+    }
+    freeaddrinfo(res0);
+    if (s == -1) {
+	msgConfirm("Couldn't connect to http://%s:%s/",
+		    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+	return NULL;
+    }
+						   
+    sprintf(req,"GET /%s/%s HTTP/1.0\r\n\r\n",
+	    variable_get(VAR_HTTP_PATH), file);
+
+    if (isDebug()) {
+	msgDebug("sending http request: %s\n",req);
+    }
+    write(s,req,strlen(req));
+
+/*
+ *  scan the headers of the response
+ *  this is extremely quick'n dirty
+ *
+ */
+    cp=buf;
+    el=FALSE;
+    rv=read(s,cp,1);
+    while (rv>0) {
+	if ((*cp == '\012') && el) {
+  	    /* reached end of a header line */
+  	    if (!strncmp(buf,"HTTP",4)) {
+		rv=strtol((char *)(buf+9),0,0);
+		*(cp-1)='\0';		/* chop the CRLF off */
+		if (probe && (rv != 200)) {
+		    return NULL;
+		} else if (rv >= 500) {
+		    msgConfirm("Server error %s when sending %s, you could try an other server",buf, req);
+		    return NULL;
+		} else if (rv == 404) {
+		    msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
+		    return NULL;
+		} else if (rv >= 400) {
+		    msgConfirm("Client error %s, you could try an other server",buf);
+		    return NULL;
+		} else if (rv >= 300) {
+		    msgConfirm("Error %s",buf);
+		    return NULL;
+		} else if (rv != 200) {
+		    msgConfirm("Error %s when sending %s, you could try an other server",buf, req);
+		    return NULL;
+		}
+	    }
+	    /* ignore other headers */
+	    /* check for "\015\012" at beginning of line, i.e. end of headers */
+	    if ((cp-buf) == 1) 
+		break;
+	    cp=buf;
+	    rv=read(s,cp,1);
+	} else {
+	    el=FALSE;
+	    if (*cp == '\015')
+		el=TRUE;
+	    cp++;
+	    rv=read(s,cp,1);
+	}
+    }
+    fp=fdopen(s,"r");
+    return fp;
+}

Property changes on: usr.sbin/sysinstall/httpdirect.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Index: usr.sbin/sysinstall/menus.c
===================================================================
--- usr.sbin/sysinstall/menus.c	(revision 248310)
+++ usr.sbin/sysinstall/menus.c	(revision 248313)
@@ -213,7 +213,8 @@
       { " Media, UFS",		"Select UFS installation media.",	NULL, mediaSetUFS },
       { " Media, FTP",		"Select FTP installation media.",	NULL, mediaSetFTP },
       { " Media, FTP Passive",	"Select passive FTP installation media.", NULL, mediaSetFTPPassive },
-      { " Media, HTTP",		"Select FTP via HTTP proxy install media.", NULL, mediaSetHTTP },
+      { " Media, HTTP Proxy",	"Select FTP via HTTP proxy install media.", NULL, mediaSetHTTP },
+      { " Media, HTTP Direct",	"Select HTTP direct installation media.", NULL, mediaSetHTTPDirect },
       { " Network Interfaces",	"Configure network interfaces",		NULL, tcpMenuSelect },
       { " Networking Services",	"The network services menu.",		NULL, dmenuSubmenu, NULL, &MenuNetworking },
       { " NFS, client",		"Set NFS client flag.",			dmenuVarCheck, dmenuToggleVariable, NULL, "nfs_client_enable=YES" },
@@ -881,6 +882,21 @@
       { NULL } }
 };
 
+DMenu MenuMediaHTTPDirect = {
+    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
+    "Please select a FreeBSD HTTP distribution site",
+    "Please select the site closest to you or \"other\" if you'd like to\n"
+    "specify a different choice.  Also note that not every site listed here\n"
+    "carries more than the base distribution kits. Only Primary sites are\n"
+    "guaranteed to carry the full range of possible distributions.",
+    "Select a site that's close!",
+    NULL,
+    { { "URL", "Specify some other ftp site by URL", NULL, dmenuSetVariable, NULL,
+	VAR_HTTP_PATH "=other" },
+
+      { NULL } }
+};
+
 DMenu MenuNetworkDevice = {
     DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
     "Network interface information required",
@@ -926,12 +942,13 @@
     { { "1 CD/DVD",		"Install from a FreeBSD CD/DVD",	NULL, mediaSetCDROM },
       { "2 FTP",		"Install from an FTP server",		NULL, mediaSetFTPActive },
       { "3 FTP Passive",	"Install from an FTP server through a firewall", NULL, mediaSetFTPPassive },
-      { "4 HTTP",		"Install from an FTP server through a http proxy", NULL, mediaSetHTTP },
-      { "5 DOS",		"Install from a DOS partition",		NULL, mediaSetDOS },
-      { "6 NFS",		"Install over NFS",			NULL, mediaSetNFS },
-      { "7 File System",	"Install from an existing filesystem",	NULL, mediaSetUFS },
-      { "8 Floppy",		"Install from a floppy disk set",	NULL, mediaSetFloppy },
-      { "9 USB",		"Install from a USB drive",		NULL, mediaSetUSB },
+      { "4 HTTP Proxy",		"Install from an FTP server through a http proxy", NULL, mediaSetHTTP },
+      { "5 HTTP Direct",	"Install from an HTTP server",		NULL, mediaSetHTTPDirect },
+      { "6 DOS",		"Install from a DOS partition",		NULL, mediaSetDOS },
+      { "7 NFS",		"Install over NFS",			NULL, mediaSetNFS },
+      { "8 File System",	"Install from an existing filesystem",	NULL, mediaSetUFS },
+      { "9 Floppy",		"Install from a floppy disk set",	NULL, mediaSetFloppy },
+      { "A USB",		"Install from a USB drive",		NULL, mediaSetUSB },
       { "X Options",		"Go to the Options screen",		NULL, optionsEditor },
       { NULL } },
 };
Index: usr.sbin/sysinstall/sysinstall.8
===================================================================
--- usr.sbin/sysinstall/sysinstall.8	(revision 248310)
+++ usr.sbin/sysinstall/sysinstall.8	(revision 248313)
@@ -684,6 +684,8 @@
 .Bl -tag -width indent
 .It _httpPath
 The proxy to use (host:port) (non-optional).
+.It httpDirectory
+The path from http root.
 .El
 .It mediaSetUFS
 Select an existing UFS partition (mounted with the label editor) as
Index: usr.sbin/sysinstall/options.c
===================================================================
--- usr.sbin/sysinstall/options.c	(revision 248310)
+++ usr.sbin/sysinstall/options.c	(revision 248313)
@@ -78,6 +78,9 @@
 	case DEVICE_TYPE_HTTP:
 	    return "HTTP Proxy";
 
+	case DEVICE_TYPE_HTTP_DIRECT:
+	    return "HTTP Direct";
+
 	case DEVICE_TYPE_CDROM:
 	    return "CDROM";

NOTE: This patch was tested with stable/8@r247902. Revisions untested are stable/9, releng/8.2 and older, releng/9.1 and older.

Update boot_crunch.conf

boot_crunch is a “crunched” binary containing compiled binaries and statically linked libraries into a single binary. boot_crunch.conf controls the contents of boot_crunch when compiling.

Replacing /usr/src/release/amd64/boot_crunch.conf with the following boot_crunch.conf causes release(7) to build a boot_crunch suitable for FreeBSD installations via ISO.

# $FreeBSD: src/release/amd64/boot_crunch.conf,v 1.67.2.1.2.1 2009/10/25 01:10:29 kensmith Exp $

buildopts -DRELEASE_CRUNCH -Dlint

srcdirs /usr/src/bin
progs hostname
progs pwd
progs rm
progs sh
progs ls
progs test
progs cat
progs df
progs sleep
ln sh -sh
ln test [

srcdirs /usr/src/sbin
progs camcontrol
special camcontrol objs camcontrol.o modeedit.o util.o
progs dhclient
progs fsck_ffs
progs ifconfig
progs mount_nfs
progs newfs
progs route
progs rtsol
progs tunefs
progs kenv
progs sysctl
progs mdmfs
progs mdconfig
progs mount
progs dmesg
progs fdisk
progs bsdlabel
ln fsck_ffs fsck_4.2bsd
ln fsck_ffs fsck_ufs

srcdirs /usr/src/usr.bin
progs uname
progs cpio
progs find
progs minigzip
progs sed
progs awk
progs fetch
progs ifconfig
ln minigzip gzip
ln minigzip gunzip
ln minigzip zcat

srcdirs /usr/src/usr.sbin
progs arp
progs sysinstall
progs usbconfig

srcdirs /usr/src/gnu/usr.bin
progs grep

libs -ll -ledit -lfetch -lmd -lcrypt -lftpio -lz -lnetgraph
libs -ldialog -lncurses -lcam -lsbuf -lutil -ldisk -lufs -ldevinfo -lkvm -lgeom
libs -lbsdxml -larchive -lbz2 -lusb -lgnuregex -lz -llzma
libs -lfetch -lmd -lssl -lcrypto -ljail -lm

NOTE: I found with r247902 and newer, the above “special” line was unnecessary and caused errors during `make release`. I am unsure when this was fixed.

make buildworld; make release

The release(7) man page describes a process for building world and making release. FreeBSD sources are a pre-requisite and can be checked out/exported from svn.freebsd.org.

There are several environment variables necessary to build a release. These are set in /usr/src/release/Makefile or on the command line. For simplicity’s sake, I’ll just set them on the command line here, but in reality, I have them set in the Makefile. The process to build world and make a release is:

# exec bash
# cd /usr/src && make buildworld
# CHROOTDIR=/usr/release CVSROOT=/home/ncvs EXTPORTSDIR=/usr/ports EXTSRCDIR=/usr/src MAKE_DVD=True NO_FLOPPIES=True NODOC=True cd release && make release

This creates compiled code in /usr/release/. Content for the bootonly ISO exists in ${CHROOTDIR}/R/cdrom/bootonly/. The mfsroot.gz necessary for the following step lives at ${CHROOTDIR}/R/cdrom/bootonly/boot/mfsroot.gz.

mfsroot.gz

mfsroot.gz is a gzip compressed BSD UFS filesystem environment containing config files, scripts, and binaries necessary for a FreeBSD 8.x install. This mfsroot.gz varies from the default as described below. Much of the below can be addressed during the release(7) process, but the main advantage of the process below is the release process does not have to be re-run for each change made to mfsroot.gz.

Devin Teske was kind enough to point me to FreeBSD Druid which is a sysinstall(8) based installation method where I was able to pick out a piece on customizing the mfsroot to use locally.

The email from from Devin:

If you want, you could follow my approach which is to take the completed mfsroot.gz and use a Makefile to manage the creation of custom mfsroots (keeping the original unmodified, making it simpler to test different iterations).

The advantage is that you don’t have to re-perform the release(7) process each time you want to make a change to your mfsroot.

Check it out:

http://druidbsd.cvs.sourceforge.net/viewvc/druidbsd/druidbsd/druid/dep/freebsd/mfsroot/standard/

Basically, you’d grab the Makefile (link below):

http://druidbsd.cvs.sourceforge.net/viewvc/druidbsd/druidbsd/druid/dep/freebsd/mfsroot/standard/Makefile?revision=1.1

Then create a “dep” directory and “src” directory:

Next, take the virgin mfsroot.gz produced by the release(7) process and dump it into the “dep” directory.

Next, put your install.cfg into the “src” directory (just like you see that I did).

Optionally populate more files into the “src” directory (see the first link above for an example — example includes “boot/modules/nullfs.ko”, “etc/fstab”, and “etc/group”, etc.).

When the “src” directory represents what you’d like to add to the mfsroot, you’re ready to produce a new copy of the stored original (at “dep/mfsroot.gz”), complete with your additions.

Execute:

make from_dep

NOTE: sudo is required

What will happen is that “dep/mfsroot.gz” will be copied to the current working directory, the mfsroot is ripped open (requires sudo privileges), the “src” directory is layered onto the mfsroot, and finally the mfsroot is packaged back up (leaving you with a custom “./mfsroot.gz” for deployment).

Copy /usr/release/R/cdrom/bootonly/boot/mfsroot.gz to an alternate location setup identically to the way Devin described above. The install.cfg, doconfig.sh, and dhclient-script are copied to src/stand/ so when `make from_dep` is executed it runs the four (4) “stage” targets that modify the mfsroot and place these files in the resulting mfsroot.gz

NOTES:
* Comment line #41 of the Makefile if the environment does not utilize FreeBSD Druid.
* A larger mfsroot.gz file is necessary if additions are greater in size than the amount of space available in the mfsroot.gz. Alternatively, modify the size of the mfsroot.gz with the value of MFSSIZE in /usr/src/release/Makefile.

Finally, copy the mfsroot.gz to /usr/release/R/cdrom/bootonly/boot/mfsroot.gz and make the ISO.

Creating New mfsroot from Existing mfsroot

This process is useful when the mfsroot.gz does not have enough free space available for any additions.

This process assumes the following:

• we add a boot_crunch file
• Resulting mfsroot is 12MB.
• Existing mfsroot mount point is /tmp/mfsroot_old
• New mfsroot mount point is /tmp/mfsroot_new
• Existing mfsroot is /tmp/mfsroot_old
• Memdisk device for the existing mfsroot is md0
• Memdisk device for the new mfsroot is md1
• New boot_crunch file is /tmp/bootcrunch/boot_crunch

# mkdir /tmp/mfsroot_old /tmp/mfsroot_new
# dd if=/dev/zero of=/tmp/mfsroot bs=1024 count=12288
# mdconfig -f /tmp/mfsroot_old; mdconfig -f /tmp/mfsroot
# newfs /dev/md1
# mount /dev/md0 /tmp/mfsroot_old; mount /dev/md1 /tmp/mfsroot_new
# cd /tmp/mfsroot_old; tar -cf - . | (cd/tmp/mfsroot_new; tar -xf -)
# cp /tmp/bootcrunch/boot_crunch /tmp/mfsroot_new/stand/
# cd /tmp/mfsroot_new/stand
# for i in $(./boot_crunch 2>&1|grep -v usage); do
    if [ "$i" != "boot_crunch" ]; then
      rm -f ./"$i"; ln ./boot_crunch "$i";
    fi
  done
# cd /; umount /tmp/mfsroot_old; umount /tmp/mfsroot_new
# mdconfig -d -u 0; mdconfig -d -u 1

/tmp/mfsroot is the new mfsroot when the process is complete. Replace the old mfsroot with this one.

install.cfg

install.cfg is sysinstall’s config file. When it is executed it checks for this file. sysinstall(8) run interactively if the file is missing, but non-interactively if it is not. This file exists in stand/ inside the mfsroot.gz.

The syntax of install.cfg is archaic and strict, but it is quite powerful if you spend the time to learn it. The file below sets up the environment where sysinstall(8) runs then executes doconfig.sh (described below).

# Turn on extra debugging.
debug=YES

# Turn off all prompting.
nonInteractive=YES
noWarn=YES

command=/bin/sh /stand/doconfig.sh
system
# Chain to the config we just downloaded
configFile=/stand/cobbler.cfg
loadConfig

doconfig.sh

This script, also in stand/ inside the mfsroot file, initializes the environment, sets variables, and communicates with Cobbler to grab the remainder of install.cfg which we see is referred to as stand/cobbler.cfg. This is described later.

#!/bin/sh
# Talk to cobbler, grab templates

DHCPLEASE=/var/empty/dhclient.leases
DHCPPID=/var/empty/dhclient.pid

# Create memdisk filesystem so we can start dhcp
mdmfs -s 32m md /var/empty

for ea in `ifconfig -l`; do
   case $ea in
      lo*) continue ;;
      plip*) continue ;;
   esac

   /stand/ifconfig $ea >> /dev/null 1&>2
   /stand/sleep 15
   /stand/dhclient -p ${DHCPPID} -l ${DHCPLEASE} $ea
done

server=`grep dhcp-server ${DHCPLEASE} | awk '{ print $3 }' | awk -F\; '{ print $1}'`;
ip=`grep fixed-address ${DHCPLEASE} | awk '{ print $2 }' | awk -F\; '{ print $1}'`;
nm=`grep subnet-mask ${DHCPLEASE} | awk '{ print $3 }' | awk -F\; '{ print $1 }'`;
gw=`grep routers ${DHCPLEASE} | awk '{ print $3 }' | awk -F\; '{ print $1 }'`;
name=`grep host-name ${DHCPLEASE} | awk '{ print $3 }' | awk -F\" '{ print $2 }'`;
route=`grep routers ${DHCPLEASE} | awk '{ print $3 }' | awk -F\; '{ print $1 }'`;
iface=`grep interface ${DHCPLEASE} | awk '{ print $2 }' | awk -F\" '{ print $2 }'`;
kspath="http://${server}/cblr/svc/op/ks/system/${name}";

# use Fetch to get my answer file from my cobbler server
# use awk to pull out different sections using "% /path/to/file" syntax.
fetch -qo - "$kspath" | awk '/^% /{f=$2} /^[^%]/ && f{print > f}'

cat > /stand/media.cfg <<EOF
netDev=${iface}
defaultrouter=${route}
ipaddr=${ip}
netmask=${nm}
EOF

cobbler.cfg

cobbler.cfg is hosted remotely (by Cobbler) and fetched via HTTP. It can be separated into sections executed as shell scripts to enable dynamic control over the configuration. The doconfig.sh script above separates the sections of the files based on “% $filename” syntax.

The $disk variables below are substituted with the actual disk identifier. This can be found using the kern.disks kernel variable.

% /stand/cobbler.cfg
# The installation media is setup in the doconfig.sh script
hostname=$system_name
configFile=/stand/media.cfg
loadConfig
_httpPath=$getVar('httppath', '')
releaseName=any
mediaSetHTTPDirect

# Select which distributions we want.
dists=base kernels GENERIC SMP doc catpages
distSetCustom

# Figure out the disk configuration
disk=${disk}
partition=all
bootManager=standard
diskPartitionEditor
${disk}s1-1=ufs 12582912 / # 6 GB root
${disk}s1-2=swap ${swap} none # swap
${disk}s1-3=ufs 2097152 /tmp 1 # tmp
${disk}s1-4=ufs 4194304 /var 1 # 2 GB var
${disk}s1-5=ufs 4194304 /home 1 # 2 GB home

# OK, everything is set.  Do it!
installCommit

package=perl
packageAdd

shutdown

It becomes necessary to use releaseName because checkAccess() appends various default directories to _httpPath in performing it’s search for the distributions when mediaSetHTTPDirect is specified.

Generate the bootonly ISO

After the release has been built and mfsroot.gz customized, it is time to create the ISO by executing the following command line syntax:

mkisofs -R -no-emul-boot -b boot/cdboot -o /FreeBSD-8.3-RELEASE-bootonly.iso $CHROOTDIR/R/cdrom/bootonly

Upload the resulting ISO to the respective Cobbler distribution where iPXE (described below) will download it from. In our example, /www/cobbler/ks_mirror/freebsd-8.3.4/boot/.

iPXE Configuration

iPXE is a bootstrap program with support for loading ISOs over http. We take the bootonly ISO generated above as well as the memdisk module and place them into the distributions boot/ directory.

iPXE loads, then downloads and executes the memdisk module (which provides iPXE the ability to load ISOs). iPXE instructs systems to either PXE boot or boot to disk based on the Cobbler system record.

gpxe_system_freebsd.template

This Cobbler template is used when “gPXE” has been enabled for a system. The system loads iPXE which in turn loads the memdisk module here.

#!gpxe
kernel http://$server/cobbler/ks_mirror/$distro/boot/memdisk
imgargs memdisk raw iso
initrd http://$server/cobbler/ks_mirror/$distro/boot/bootonly.iso.zip
boot

$server and $distro are dynamically templated out by Cobbler when the netboot enabled system PXE boots.

Cobbler Configurations

The following configuration are necessary in Cobbler:

• Set httppath in the kickstart metadata for the distro
• Set gPXE to enabled/true for each Cobbler system record necessary
• Set netboot enabled to the appropriate configuration
• Cobbler patches

httppath Kickstart Metadata

The distro must be imported into Cobbler before configuring this. The variable is set to the URI of the FreeBSD media. In our example, the distro is imported to /www/cobbler/ks_mirror/freebsd-8.3.4/. Thus necessitating the need to set the variable as follows (observe the image below):

httppath=http://@@http_server@@/cobbler/ks_mirror/freebsd-8.3.4/8.3-RELEASE-p4

gPXE Configuration

Systems or machines that may require re-installation should have the “Enable gPXE” option set to “True” or “Enabled”. This is accomplished via the Cobbler web or command line interfaces. Observe this image:

Netboot Configuration

A system record whose netboot option is set to enabled/true is expected to PXE boot and install the OS. Manipulating this option is done via the Cobbler web or command line interfaces. Observe:

Cobbler Patches

A colleague wrote these patches for Cobbler 2.4.0 which allow us to boot into iPXE, load memdisk and the bootonly.iso. They have been submitted to the Cobbler project.

These patches add support for:

• Local boot patch
• gpxe/ipxe support for FreeBSD

DHCP Configuration

Cobbler can be used to manage DHCP, which is how we have done it. Cobbler manages our DHCP configuration based on the DHCP template below, typically stored at /etc/cobbler/dhcp.template. Cobbler uses the cheetah templating language to produce a valid dhcpd.conf file. Review the template below.

NOTE: A colleague has developed a few patches for Cobbler which he intends to submit back to the Cobbler community.

Cobbler DHCP Template

# ******************************************************************
# Cobbler managed dhcpd.conf file
#
# generated from cobbler dhcp.conf template ($date)
# Do NOT make changes to /etc/dhcpd.conf. Instead, make your changes
# in /etc/cobbler/dhcp.template, as /etc/dhcpd.conf will be
# overwritten.
#
# ******************************************************************

ddns-update-style interim;

allow booting;
allow bootp;

ignore client-updates;
set vendorclass = option vendor-class-identifier;

subnet 192.168.2.0 netmask 255.255.255.0 {
     option routers             192.168.2.1;
     option ntp-servers		192.168.2.1;
     option domain-name-servers 192.168.2.20, 192.168.3.20;
     option subnet-mask         255.255.255.0;
     pool {
	  range 192.168.2.50 192.168.2.100;
     }
     filename                   "/pxelinux.0";
     default-lease-time         21600;
     max-lease-time             43200;
     next-server                $next_server;
}

#for dhcp_tag in $dhcp_tags.keys():
    ## group could be subnet if your dhcp tags line up with your subnets
    ## or really any valid dhcpd.conf construct ... if you only use the
    ## default dhcp tag in cobbler, the group block can be deleted for a
    ## flat configuration
# group for Cobbler DHCP tag: $dhcp_tag
group {
        #for mac in $dhcp_tags[$dhcp_tag].keys():
            #set iface = $dhcp_tags[$dhcp_tag][$mac]
    host $iface.name_iface {
        hardware ethernet $mac;
        #if $iface.ip_address:
        fixed-address $iface.ip_address;
        #end if
        #if $iface.hostname:
        #option host-name "$iface.hostname";
        option host-name "$iface.name";
        #end if
        #if $iface.netmask:
        option subnet-mask $iface.netmask;
        #end if
        #if $iface.gateway:
        #option routers $iface.gateway;
        #end if
        #if $iface.enable_gpxe:
        if exists user-class and option user-class = "iPXE" {
            filename "http://$cobbler_server/cblr/svc/op/gpxe/system/$iface.owner";
        } else {
            filename "/undionly.kpxe";
        }
        #else
        filename "$iface.filename";
        #end if
        ## Cobbler defaults to $next_server, but some users
        ## may like to use $iface.system.server for proxied setups
        next-server $next_server;
        ## next-server $iface.next_server;
    }
        #end for
}
#end for

DHCP Configuration File

Below is an example of a dhcp entry in dhcpd.conf

    host testbsd {
        hardware ethernet xx:xx:xx:xx:xx:xx;
        fixed-address 192.168.0.5;
        option host-name "testbsd";
        option subnet-mask 255.255.255.0;
        if exists user-class and option user-class = "iPXE" {
            filename "http://192.168.0.2/cblr/svc/op/gpxe/system/testbsd";
        } else {
            filename "/undionly.kpxe";
        }
        next-server 192.168.0.2;

Disclaimer

This blog is posted for informational purposes only. Extensive testing is recommended prior to implementing changes discussed here.