Merge branch 'next' into datarewrite-structures

25 files changed, 442 insertions, 52 deletions

diff --git a/doc/ref/general.texi b/doc/ref/general.texi
new file mode 100644
index 00000000..b97dece0
--- /dev/null
+++ b/doc/ref/general.texi
@@ -0,0 +1,31 @@
+@node Generally Useful Utilities
+@chapter Generally Useful Utilities
+
+These are all general utilitios with no direct relation to Calp. Many
+of these should be generally useful in any project.
+
+@include general/datetime.texi
+@include general/zic.texi
+@include general/srfi-41.texi
+@include general/util.texi
+@include general/uuid.texi
+@include general/io.texi
+@include general/bst.texi
+@include general/util-path.texi
+@include general/util-config.texi
+@include general/util-exceptions.texi
+@include general/base64.texi
+@include general/xdg-basedir.texi
+@include general/glob.texi
+@include general/graphviz.texi
+@include general/crypto.texi
+@include general/graph.texi
+@include general/options.texi
+
+
+@include general/util-type.texi
+@include general/util-object.texi
+@include general/lens.texi
+@include general/data-formats.texi
+@include general/data-stores.texi
+@include general/webdav.texi
diff --git a/doc/ref/guile/base64.texi b/doc/ref/general/base64.texi
index ab6bba81..ab6bba81 100644
--- a/doc/ref/guile/base64.texi
+++ b/doc/ref/general/base64.texi
diff --git a/doc/ref/general/bst.texi b/doc/ref/general/bst.texi
new file mode 100644
index 00000000..d2bba0ff
--- /dev/null
+++ b/doc/ref/general/bst.texi
@@ -0,0 +1,37 @@
+@node Binary Search Tree
+@section Binary Search Tree
+
+A simple ``read only'' binary search tree.
+
+@defun make-tree pred? lst
+Constructs a new tree. @var{pred?} should be a procedure taking the
+first element of @var{lst}, along with each element, and should return
+a boolean value indicating if the specific element should go in the
+left or right subtree. (left subtree is ``truthy'' values).
+
+This operation is done recursively.
+@end defun
+
+@defun tree-node tree
+Return the value of a tree node.
+@end defun
+
+@defun left-subtree tree
+Return all ``truthy'' children of tree node.
+@end defun
+
+@defun right-subtree tree
+Return all ``falsy children of tree node.
+@end defun
+
+@defun length-of-longest-branch tree
+Get the depth of a tree.
+@end defun
+
+@defun tree-map proc tree
+Apply proc onto the value of every node in tree, keeping the structure
+of the tree.
+
+@b{Note:} this can cause the tree to no longer be a binary search
+tree, but simply a ``formless'' binary tree.
+@end defun
diff --git a/doc/ref/general/crypto.texi b/doc/ref/general/crypto.texi
new file mode 100644
index 00000000..b9e362d3
--- /dev/null
+++ b/doc/ref/general/crypto.texi
@@ -0,0 +1,17 @@
+@node Cryptographic and Hash Procedures
+@section Cryptographic and Hash Procedures
+
+This module links libcrypto, exposing some hash procedures.
+
+@defun sha256 message
+Calculate the sha256-sum of @var{message}. The message can either be a
+bytevector or a string (in which case it will de encoded as UTF-8).
+
+Returns the checksum as a bytevector.
+@end defun
+
+@defun checksum->string message-digest [port]
+Generates a hex digest string from a checksum (generated by
+@code{sha256}). The checksum is written to @var{port} if given, or
+returned as a string otherwise.
+@end defun
diff --git a/doc/ref/guile/data-formats.texi b/doc/ref/general/data-formats.texi
index 037d3ae7..037d3ae7 100644
--- a/doc/ref/guile/data-formats.texi
+++ b/doc/ref/general/data-formats.texi
diff --git a/doc/ref/guile/data-stores.texi b/doc/ref/general/data-stores.texi
index ec3962da..ec3962da 100644
--- a/doc/ref/guile/data-stores.texi
+++ b/doc/ref/general/data-stores.texi
diff --git a/doc/ref/guile/datetime.texi b/doc/ref/general/datetime.texi
index 037ac8d5..037ac8d5 100644
--- a/doc/ref/guile/datetime.texi
+++ b/doc/ref/general/datetime.texi
diff --git a/doc/ref/general/glob.texi b/doc/ref/general/glob.texi
new file mode 100644
index 00000000..400eb1f7
--- /dev/null
+++ b/doc/ref/general/glob.texi
@@ -0,0 +1,37 @@
+@node Glob
+@section Glob
+
+@defun glob str
+Globs (glob(7)) on @var{str}, returing a list of files, which will be
+on the same form of the glob. E.g. @code{(glob "../*")} will return
+strings all starting with ``../''
+
+If no matches are found, a misc error is thrown.
+@end defun
+
+
+
+@defvar GLOB_NOMAGIC
+@defvarx GLOB_NOCHECK
+@defvarx unix
+@defvarx GLOB_NOSPACE
+@defvarx GLOB_TILDE_CHECK
+@defvarx GLOB_ALTDIRFUNC
+@defvarx GLOB_NOSORT
+@defvarx GLOB_NOMATCH
+@defvarx GLOB_TILDE
+@defvarx GLOB_ERR
+@defvarx GLOB_MAGCHAR
+@defvarx GLOB_BRACE
+@defvarx GLOB_APPEND
+@defvarx GLOB_NOSYS
+@defvarx GLOB_DOOFFS
+@defvarx GLOB_NOESCAPE
+@defvarx GLOB_MARK
+@defvarx GLOB_PERIOD
+@defvarx linux
+@defvarx GLOB_ABORTED
+@defvarx _POSIX_VDISABLE
+@defvarx GLOB_ONLYDIR
+``Symbols'' imported from the C header ``glob.h''. See that documentation.
+@end defvar
diff --git a/doc/ref/general/graph.texi b/doc/ref/general/graph.texi
new file mode 100644
index 00000000..1677b635
--- /dev/null
+++ b/doc/ref/general/graph.texi
@@ -0,0 +1,65 @@
+@node A Graph data structure
+@section A Graph data structure
+
+This is a simple immutable directed graph.
+
+Most operations are O(n), since Scheme lacks a total order betwen
+arbitrary objects.
+
+@defun make-graph [node-key-proc=identity] [node-equal?=eq?]
+Constructs a new graph.
+
+@var{node-key-proc} should be a procedure mapping the actual nodes to
+a value which can be compared. These values are sometimes called
+``node keys''.
+@cindex node keys
+
+@var{node-equal?} should return if two nodes are equal or not.
+@end defun
+
+@defun rebuild-graph [old-graph] [nodes='()] [edges='()]
+@c TODO document me
+@end defun
+
+@defun graph-empty? graph
+Does the graph contaitn
+@end defun
+
+@defun add-node graph node edge-neighbors
+Adds the value @var{node} to @var{graph}. @var{edge-neighbors} should
+be a list of node keys.
+@end defun
+
+@defun get-node graph key
+Retrieve a node from the graph, given its node key.
+
+Returns @code{#f} if no such node exists.
+@end defun
+
+@defun remove-node graph node
+Remvoe @var{node} from @var{graph}, if it exists.
+@end defun
+
+@defun find-dangling-node graph
+Find a node in the graph which no other node ``depends'' on (has an
+edge pointing at it).
+
+NOTE this is O(n^2) (maybe, sort of?)
+Getting it faster would require building an index, which
+is hard since there isn't a total order on symbols.
+@end defun
+
+
+@defun pop-dandling-node graph
+Find a node as per @code{find-dangling-node}, and return two values:
+the node, and a new graph without that node.
+@end defun
+
+@defun resolve-dependency-graph graph
+If each edge is assumed to indicate a nodes dependencies, then this
+procedure will find return a flat list where each dependency is before
+its dependants.
+
+If any link is missing, or a cycle exists, then the result is
+undefined.
+@end defun
diff --git a/doc/ref/general/graphviz.texi b/doc/ref/general/graphviz.texi
new file mode 100644
index 00000000..72817ea8
--- /dev/null
+++ b/doc/ref/general/graphviz.texi
@@ -0,0 +1,10 @@
+@node Graphviz
+@section Graphviz
+
+The graphviz library comes bundled with Guile bindings, but without a
+corresponding .scm file exporting the symbols. The module @code{(guile)} does
+exactly that.
+
+This ``header'' is borrowed from
+@url{https://github.com/roelj/graphviz-guile/blob/master/graphviz.scm},
+under GPL 3.0.
diff --git a/doc/ref/general/io.texi b/doc/ref/general/io.texi
new file mode 100644
index 00000000..3f67700b
--- /dev/null
+++ b/doc/ref/general/io.texi
@@ -0,0 +1,31 @@
+@node IO operations
+@section IO operations
+
+Provided by module @code{(hnh util io)}.
+
+@defun open-input-port path
+@defunx open-output-port path
+Like @code{open-*-file}, but ``-'' gives @code{standard-@{input,output@}}.
+@end defun
+
+@defun read-lines port
+Return a list of all lines read from port.
+@end defun
+
+@defun with-atomic-output-to-file filename thunk
+Same functionality as the regular @var{with-output-to-file}, but
+with the difference that either everything is written, or nothing
+is written, and if anything is written it's all written atomicaly at
+once (the original file will never contain an intermidiate state).
+Does NOT handle race conditions between threads.
+
+propagates the return value of @var{thunk} upon successfully writing
+the file, and @code{#f} otherwise.
+@end defun
+
+@defun call-with-tmpfile proc [#:tmpl ``/tmp/file-XXXXXXX'']
+@end defun
+
+@defun read-file path
+Open file at path, and return its content as a string.
+@end defun
diff --git a/doc/ref/guile/lens.texi b/doc/ref/general/lens.texi
index eeddd6ca..eeddd6ca 100644
--- a/doc/ref/guile/lens.texi
+++ b/doc/ref/general/lens.texi
diff --git a/doc/ref/general/options.texi b/doc/ref/general/options.texi
new file mode 100644
index 00000000..675e91f3
--- /dev/null
+++ b/doc/ref/general/options.texi
@@ -0,0 +1,32 @@
+@node Getopt-Long Extensions
+@section Getopt-Long extensions
+
+The module @code{(hnh util options)} extend Guile's
+@code{(ice-9 getopt-long)}. The extra keys which can be specified on
+an option are
+
+@table @samp
+@item (description string)
+A propper description of the value. Should be written for human
+consumption.
+
+@item (value value-spec)
+Value is from the core library, but here ...
+@c TODO document me
+@end table
+
+@defun getopt-opt options
+Remove extensions from @var{options}, turning it into a structure
+which can be passed to @code{getopt-long}.
+@end defun
+
+@defun format-arg-help options
+Pretty print an option summary.
+
+Each description will be parsed as XML and then fed through our markup
+system @xref{Markup}.
+@end defun
+
+@defun print-arg-help options [port=(current-error-port)]
+Formats and prints option spec.
+@end defun
diff --git a/doc/ref/guile/srfi-41.texi b/doc/ref/general/srfi-41.texi
index 8c65b6eb..d8020ecc 100644
--- a/doc/ref/guile/srfi-41.texi
+++ b/doc/ref/general/srfi-41.texi
@@ -39,7 +39,7 @@ was found.
 @end defun
 
 
-@defun stream-remave pred stream
+@defun stream-remove pred stream
 Stream-filter, but with predicate negated.
 @end defun
 
@@ -73,6 +73,26 @@ times.
 stream cons, but eval arguments beforehand.
 @end defun
 
+@defun stream-split-by pred strm
+Chunks the content of @var{strm} into lists, breaking on @var{pred}.
+If the end of the stream is reached, the remaining objects
+are put into a final chunk.
+
+Can for example be used to split a stream of characters into a stream
+of words.
+
+@lisp
+(stream-split-by (lambda (c) (char=? c #\space))
+                 (-> "This is a short test"
+                     string->list list->stream))
+⇒ #<stream (#\T #\h #\i #\s #\space)
+           (#\i #\s #\space)
+           (#\a #\space)
+           (#\s #\h #\o #\r #\t #\space)
+           (#\t #\e #\s #\t)>
+@end lisp
+@end defun
+
 @defun stream-timeslice-limit stream timeslice
 Wrap a stream in time limits. Each element has at most @var{timeslice}
 seconds to produce a value, otherwise the stream ends. Useful for finding the
diff --git a/doc/ref/guile/sxml.texi b/doc/ref/general/sxml.texi
index dd635b4c..dd635b4c 100644
--- a/doc/ref/guile/sxml.texi
+++ b/doc/ref/general/sxml.texi
diff --git a/doc/ref/guile/util-config.texi b/doc/ref/general/util-config.texi
index 2e197bcc..2e197bcc 100644
--- a/doc/ref/guile/util-config.texi
+++ b/doc/ref/general/util-config.texi
diff --git a/doc/ref/general/util-exceptions.texi b/doc/ref/general/util-exceptions.texi
new file mode 100644
index 00000000..34ba33f9
--- /dev/null
+++ b/doc/ref/general/util-exceptions.texi
@@ -0,0 +1,40 @@
+@node Exception & Warning Utilities
+@section Exception & Warning Utilities
+@anchor{warning}
+
+Warnings are like exceptions, but only fatal when wanted.
+
+@code{(hnh util exceptions)}
+
+@defun warning fmt args ...
+Emit a warning.
+
+If the parameter @var{warnings-are-errors} is true, then an exception
+of type @code{'warning} will be raised, with @var{fmt} and @var{args}
+as arguments.
+
+If that parameter is false, then the procedure in
+@var{warning-handler} will instead be called.
+@end defun
+
+@deftp {parameter} warning-handler
+Parameter containing proceudre which will be called for non-throwing
+warnings. This procedure is assumed by the program to log the warning
+in some way, and continue program flow. But anything goes.
+
+The procedure is given a format specifier (as per Scheme's basic
+@code{format}), along with the correct number of arguments.
+@end deftp
+
+@deftp {parameter} warnings-are-errors
+Boolean parameter, which if set causes warnings to be thrown as exceptions.
+@end deftp
+
+@defun fatal fmt args ...
+Display the message in fmt, populated with args, then raises the UNIX
+signal SIGINT, which kills the program.
+@end defun
+
+@defun filter-stack pred? stack
+@c TODO document me
+@end defun
diff --git a/doc/ref/guile/util-object.texi b/doc/ref/general/util-object.texi
index ceac2f2a..ceac2f2a 100644
--- a/doc/ref/guile/util-object.texi
+++ b/doc/ref/general/util-object.texi
diff --git a/doc/ref/guile/util-path.texi b/doc/ref/general/util-path.texi
index 9cf41b40..ba78a828 100644
--- a/doc/ref/guile/util-path.texi
+++ b/doc/ref/general/util-path.texi
@@ -1,14 +1,19 @@
 @node Path Utilities
 @section Path Utilities
 
-Provided by the module @code{(hnh util path)}.
+An extended path library for Guile. This library builds upon the path
+utilities provided by @xref{File System,,,guile}, but adds
+manipulation procedures. These procedures should work no matter what
+the systems directory delimiter is, even though this documentation
+uses ``/'' for its examples.
 
+Provided by the module @code{(hnh util path)}.
 
 @defun path-absolute? string
 Alias of @code{absolute-file-name?} from Guile.
 @end defun
 
-@defun path-append strings ...
+@defun path-append path paths ...
 Joins all strings into a path, squeezing duplicated delimiters, but
 ensuring that all delimiters that are needed are there.
 
diff --git a/doc/ref/guile/util-type.texi b/doc/ref/general/util-type.texi
index 104b00b3..104b00b3 100644
--- a/doc/ref/guile/util-type.texi
+++ b/doc/ref/general/util-type.texi
diff --git a/doc/ref/guile/util.texi b/doc/ref/general/util.texi
index 1d35e0bf..1d6a4e7a 100644
--- a/doc/ref/guile/util.texi
+++ b/doc/ref/general/util.texi
@@ -1,5 +1,8 @@
-@node General Utilities
-@section General Utilities
+@node Miscellaneous utilities
+@section Miscellaneous utilities
+
+A kitchen sink utility library. Note should be taken that some core
+procedures and forms are replaced, but all in compatible ways.
 
 Provided by the module @code{(hnh util)}.
 
@@ -34,8 +37,8 @@ our extra specialized @var{when}}, but binds the return of
 @defmacx for (key ...) in collection body ...
 Syntactic sugar over @code{map}.
 @example
-for x in collection
-    body ...
+(for x in collection
+     body ...)
 @end example
 expands into
 @example
@@ -44,6 +47,17 @@ expands into
 
 If keys are a list, an match-lambda is used instead.
 @xref{Pattern Matching,,,guile}
+
+@defun break args ...
+Abandon the entire loop. Returing what was given to @code{break}.
+@end defun
+
+@defun continue [arg]
+Abandon the current iteration of the loop. If an argument is given,
+it's used as the result in the resulting list, otherwise @code{#f} is
+used.
+@end defun
+
 @end defmac
 
 
@@ -322,6 +336,9 @@ Converts @var{any} to a string, as per @var{display}.
 @defmac let-env bindings body ...
 Similar to @var{let}, but sets environment variables for the code in
 body. Restores the old values once we leave.
+
+A variable can also be removed from the environment, by setting its
+value to @code{#f}.
 @end defmac
 
 @defmac with-locale1 category locale thunk
@@ -338,51 +355,20 @@ innermost.
 then @code{with-throw-handler} is used instead of @code{catch}.
 @end defmac
 
-@subsection UUID generation
+@defun uniq lst
+@defunx univ lst
+@defunx unique lst
+@defunx uniqx comp lst
+Squash repeated equivalent elements in a list into single instances,
+similar to the POSIX command uniq(1). The three variants uses
+@code{eq?}, @code{eqv?}, and @code{equal?} respectively.
 
-Provided by module @code{(hnh util uuid)}.
-
-@defun uuid-v4
-Generates a UUID-v4 string.
-@end defun
-
-@defun uuid
-Generates an implementation defined (but guaranteed valid) UUID.
+@code{uniqx} also takes @var{comp}, which is sholud be a binary
+procedure returning if the elements are equal.
 @end defun
 
-@subsection IO
-
-Provided by module @code{(hnh util io)}.
-
-@defun open-input-port path
-@defunx open-output-port path
-Like @code{open-*-file}, but ``-'' gives @code{standard-@{input,output@}}.
-@end defun
-
-@defun read-lines port
-Return a list of all lines read from port.
-@end defun
-
-@defun with-atomic-output-to-file filename thunk
-Same functionality as the regular @var{with-output-to-file}, but
-with the difference that either everything is written, or nothing
-is written, and if anything is written it's all written atomicaly at
-once (the original file will never contain an intermidiate state).
-Does NOT handle race conditions between threads.
-
-propagates the return value of @var{thunk} upon successfully writing
-the file, and @code{#f} otherwise.
-@end defun
-
-@defun call-with-tmpfile proc [#:tmpl ``/tmp/file-XXXXXXX'']
-@end defun
-
-@defun ->port port-or-strings
-If @var{port-or-string} is a port, return it directly. If it's a
-string, instead return an input string containing the strings content.
-@end defun
-
-@c Is this even a procedure?
-@defun read-file path
-Open file at path, and return its content as a string.
-@end defun
+@defmac begin1 forms ...
+Port of Common Lisp's @code{begin1} form. Like @code{begin} runs each
+form in its body in order, but returns the first result instead of the
+last.
+@end defmac
diff --git a/doc/ref/general/uuid.texi b/doc/ref/general/uuid.texi
new file mode 100644
index 00000000..78674763
--- /dev/null
+++ b/doc/ref/general/uuid.texi
@@ -0,0 +1,17 @@
+@node UUIDs
+@section UUID generation
+
+Provided by module @code{(hnh util uuid)}.
+
+@defun uuid-v4
+Generates a UUID-v4 string.
+@end defun
+
+@defun uuid
+Generates an implementation defined (but guaranteed valid) UUID.
+@end defun
+
+@deftp {parameter} seed
+Guile parameter containing the seed used when generating UUID's in
+this module. Only set this when you want non-random randomness.
+@end deftp
diff --git a/doc/ref/guile/webdav.texi b/doc/ref/general/webdav.texi
index a495c945..a495c945 100644
--- a/doc/ref/guile/webdav.texi
+++ b/doc/ref/general/webdav.texi
diff --git a/doc/ref/general/xdg-basedir.texi b/doc/ref/general/xdg-basedir.texi
new file mode 100644
index 00000000..2d3b2972
--- /dev/null
+++ b/doc/ref/general/xdg-basedir.texi
@@ -0,0 +1,62 @@
+@node XDG Base Directory
+@section XDG Base Directory
+
+Implementation of the XDG Base Directory Specification
+@url{https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html}, v0.8.
+
+It's recommended to include this module with a symbol prefix:
+
+@example
+(use-modules ((xdg basedir) :prefix xdg-)
+
+(xdg-data-dirs)
+⇒ ("/usr/local/share" "/usr/share")
+@end example
+
+The users home directory is fetched from the environment variable @env{HOME}.
+
+
+@defun data-home
+@findex XDG_DATA_HOME
+@env{XDG_DATA_HOME}, usually @file{~/.local/share}.
+@end defun
+
+@defun config-home
+@findex XDG_CONFIG_HOME
+@env{XDG_CONFIG_HOME}, usually @file{~/.config}.
+@end defun
+
+@defun state-home
+@findex XDG_STATE_HOME
+@env{XDG_STATE_HOME}, usually @file{~/.local/state}.
+@end defun
+
+@defun data-dirs
+@findex XDG_DATA_DIRS
+@env{XDG_DATA_DIRS}, split into a list.
+
+Defaults to @code{("/usr/local/share" "/usr/share")}.
+@end defun
+
+@defun config-dirs
+@findex XDG_CONFIG_DIRS
+@env{XDG_CONFIG_DIRS} as a list, usually @code{("/etc/xdg")}.
+
+Defaults to @file{/etc/xdg}.
+@end defun
+
+@defun cache-home
+@findex XDG_CACHE_HOME
+@env{XDG_CACHE_HOME}, usually @file{~/.cache}.
+@end defun
+
+@defun runtime-dir
+@findex XDG_RUNTIME_DIR
+If @env{XDG_RUNTIME_DIR} is set, than that is used. Otherwise a
+warning message is printed to stderr and @file{/tmp} is returned.
+
+The standard also stipulates a few things about permissons for this
+directory. These are currently not checked.
+
+Systemd usually sets this value to @file{/run/user/$(id -u)}.
+@end defun
diff --git a/doc/ref/guile/zic.texi b/doc/ref/general/zic.texi
index 5af36fd3..5af36fd3 100644
--- a/doc/ref/guile/zic.texi
+++ b/doc/ref/general/zic.texi