Update to latest Turtle test suite.

Support UTF-8 in blank node names.
Support idiotic SPARQL escaping in local names.


git-svn-id: http://svn.drobilla.net/serd/trunk@452 490d8e77-9747-427b-9fa3-0b8f29cee8a0

NEWS

@@ -1,14 +1,14 @@
 serd (0.18.3) unstable;
 
-  * Support most of the latest Turtle Draft
+  * Support the latest Turtle Draft
   * Don't write xsd:decimal literals to Turtle bare if they would not be read
     back with the same type
   * Fix possible crash in serd_writer_end_anon() when writing invalid lists
   * Generate blank names like _:b1 and _:B2 not _:genid1 _:docid2
   * Correctly handle posix_memalign failure
-  * Update to waf 1.7.11
+  * Update to waf 1.7.14
 
- -- David Robillard <d@drobilla.net>  Tue, 20 Aug 2013 11:33:48 -0400
+ -- David Robillard <d@drobilla.net>  Tue, 24 Dec 2013 16:34:13 -0500
 
 serd (0.18.2) stable;
 

src/reader.c

@@ -615,7 +615,10 @@ static bool
 read_PN_CHARS_BASE(SerdReader* reader, Ref dest)
 {
 	const uint8_t c = peek_byte(reader);
-	if (is_alpha(c)) {	// TODO: UTF-8
+	if ((c & 0x80)) {  // Multi-byte character
+		return !read_utf8_character(reader, dest, eat_byte_safe(reader, c));
+	}
+	if (is_alpha(c)) {
 		push_byte(reader, dest, eat_byte_safe(reader, c));
 		return true;
 	}
@@ -626,7 +629,11 @@ static bool
 read_PN_CHARS(SerdReader* reader, Ref dest)
 {
 	const uint8_t c = peek_byte(reader);
-	if (is_alpha(c) || is_digit(c) || c == '_' || c == '-') {  // TODO: UTF-8
+	if ((c & 0x80)) {  // Multi-byte character
+		return !read_utf8_character(reader, dest, eat_byte_safe(reader, c));
+	}
+	
+	if (is_alpha(c) || is_digit(c) || c == '_' || c == '-') {
 		push_byte(reader, dest, eat_byte_safe(reader, c));
 		return true;
 	}

src/writer.c

@@ -137,7 +137,9 @@ copy_node(SerdNode* dst, const SerdNode* src)
 static inline size_t
 sink(const void* buf, size_t len, SerdWriter* writer)
 {
-	if (writer->style & SERD_STYLE_BULK) {
+	if (len == 0) {
+		return 0;
+	} else if (writer->style & SERD_STYLE_BULK) {
 		return serd_bulk_sink_write(buf, len, &writer->bulk_sink);
 	} else {
 		return writer->sink(buf, len, writer->stream);
@@ -171,7 +173,7 @@ parse_utf8_char(SerdWriter* writer, const uint8_t* utf8, size_t* size)
 	uint8_t in = utf8[i++];
 	
 #define READ_BYTE() \
-	in = utf8[i++] & 0x3f; \
+	in = utf8[i++] & 0x3F; \
 	c  = (c << 6) | in;
 
 	switch (*size) {
@@ -242,21 +244,62 @@ write_uri(SerdWriter* writer, const uint8_t* utf8, size_t n_bytes)
 			}
 		}
 
-		if (j > i) {
-			// Bulk write all characters up to this special one
-			len += sink(&utf8[i], j - i, writer);
-			i = j;
-			continue;
+		// Bulk write all characters up to this special one
+		len += sink(&utf8[i], j - i, writer);
+ 		if ((i = j) == n_bytes) {
+			break;  // Reached end
 		}
 
 		// Write UTF-8 character
 		size_t size = 0;
 		len += write_character(writer, utf8 + i, &size);
 		i   += size;
+	}
+	return len;
+}
 
-		if (size == 0) {
-			return len;
+static bool
+lname_must_escape(const uint8_t c)
+{
+	/* This arbitrary list of characters, most of which have nothing to do with
+	   Turtle, must be handled as special cases here because the RDF and SPARQL
+	   WGs are apparently intent on making the once elegant Turtle a baroque
+	   and inconsistent mess, throwing elegance and extensibility completely
+	   out the window for no good reason.
+
+	   Note '-', '.', and '_' are also in PN_LOCAL_ESC, but are valid unescaped
+	   in local names, so they are not escaped here. */
+
+	switch (c) {
+	case '\'': case '!': case '#': case '$': case '%': case '&':
+	case '(': case ')': case '*': case '+': case ',': case '/':
+	case ';': case '=': case '?': case '@': case '~':
+		return true;
+	}
+	return false;
+}
+
+static size_t
+write_lname(SerdWriter* writer, const uint8_t* utf8, size_t n_bytes)
+{
+	size_t len = 0;
+	for (size_t i = 0; i < n_bytes; ++i) {
+		size_t j = i;  // Index of next character that must be escaped
+		for (; j < n_bytes; ++j) {
+			if (lname_must_escape(utf8[j])) {
+				break;
+			}
 		}
+
+		// Bulk write all characters up to this special one
+		len += sink(&utf8[i], j - i, writer);
+ 		if ((i = j) == n_bytes) {
+			break;  // Reached end
+		}
+
+		// Write escape
+		len += sink("\\", 1, writer);
+		len += sink(&utf8[i], 1, writer);
 	}
 	return len;
 }
@@ -276,10 +319,9 @@ write_text(SerdWriter* writer, TextContext ctx,
 			}
 		}
 
-		if (j > i) {
-			len += sink(&utf8[i], j - i, writer);
-			i = j;
-			continue;
+		len += sink(&utf8[i], j - i, writer);
+		if ((i = j) == n_bytes) {
+			break;  // Reached end
 		}
 
 		uint8_t in = utf8[i++];
@@ -447,7 +489,7 @@ write_node(SerdWriter*        writer,
 			sink(">", 1, writer);
 			break;
 		case SERD_TURTLE:
-			sink(node->buf, node->n_bytes, writer);
+			write_lname(writer, node->buf, node->n_bytes);
 		}
 		break;
 	case SERD_LITERAL:

tests/TurtleTests/LICENSE

@@ -0,0 +1,117 @@
+Summary
+=======
+
+Distributed under both the W3C Test Suite License[1] and the W3C 3-clause BSD
+License[2]. To contribute to a W3C Test Suite, see the policies and contribution
+forms [3]
+
+1. http://www.w3.org/Consortium/Legal/2008/04-testsuite-license
+2. http://www.w3.org/Consortium/Legal/2008/03-bsd-license
+3. http://www.w3.org/2004/10/27-testcases
+
+DISCLAIMER
+
+UNDER BOTH MUTUALLY EXCLUSIVE LICENSES, THIS DOCUMENT AND ALL DOCUMENTS, TESTS
+AND SOFTWARE THAT LINK THIS STATEMENT ARE PROVIDED "AS IS," AND COPYRIGHT
+HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING,
+BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE, NON-INFRINGEMENT, OR TITLE; THAT THE CONTENTS OF THE DOCUMENT ARE
+SUITABLE FOR ANY PURPOSE; NOR THAT THE IMPLEMENTATION OF SUCH CONTENTS WILL NOT
+INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS.
+COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE DOCUMENT OR THE PERFORMANCE
+OR IMPLEMENTATION OF THE CONTENTS THEREOF.
+
+
+W3C Test Suite License
+======================
+
+This document, Test Suites and other documents that link to this statement are
+provided by the copyright holders under the following license: By using and/or
+copying this document, or the W3C document from which this statement is linked,
+you (the licensee) agree that you have read, understood, and will comply with
+the following terms and conditions:
+
+Permission to copy, and distribute the contents of this document, or the W3C
+document from which this statement is linked, in any medium for any purpose and
+without fee or royalty is hereby granted, provided that you include the
+following on ALL copies of the document, or portions thereof, that you use:
+
+1 A link or URL to the original W3C document.
+
+2 The pre-existing copyright notice of the original author, or if it doesn't
+  exist, a notice (hypertext is preferred, but a textual representation is
+  permitted) of the form: "Copyright © [$date-of-document] World Wide Web
+  Consortium, (Massachusetts Institute of Technology, European Research
+  Consortium for Informatics and Mathematics, Keio University) and others. All
+  Rights
+  Reserved. http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html"
+
+3 If it exists, the STATUS of the W3C document.
+
+4 When space permits, inclusion of the full text of this NOTICE should be
+  provided. We request that authorship attribution be provided in any software,
+  documents, or other items or products that you create pursuant to the
+  implementation of the contents of this document, or any portion thereof.
+
+
+No right to create modifications or derivatives of W3C documents is granted
+pursuant to this license. However, if additional requirements (documented in the
+Copyright FAQ) are satisfied, the right to create modifications or derivatives
+is sometimes granted by the W3C to individuals complying with those
+requirements.
+
+If a Test Suite distinguishes the test harness (or, framework for navigation)
+and the actual tests, permission is given to remove or alter the harness or
+navigation if the Test Suite in question allows to do so. The tests themselves
+shall NOT be changed in any way.
+
+The name and trademarks of W3C and other copyright holders may NOT be used in
+advertising or publicity pertaining to this document or other documents that
+link to this statement without specific, written prior permission. Title to
+copyright in this document will at all times remain with copyright
+holders. Permission is given to use the trademarked string W3C within claims of
+performance concerning W3C Specifications or features described therein, and
+there only, if the test suite so authorizes.
+
+THIS WORK IS PROVIDED BY W3C, MIT, ERCIM, KEIO UNIVERSITY, THE COPYRIGHT HOLDERS
+AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL W3C, MIT, ERCIM, KEIO
+UNIVERSITY, THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+W3C 3-clause BSD License
+========================
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1 Redistributions of works must retain the original copyright notice, this list
+  of conditions and the following disclaimer.
+
+2 Redistributions in binary form must reproduce the original copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+3 Neither the name of the W3C nor the names of its contributors may be used to
+  endorse or promote products derived from this work without specific prior
+  written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

tests/TurtleTests/LITERAL1_with_UTF8_boundaries.ttl

@@ -1 +1 @@
-<http://a.example/s> <http://a.example/p> '€߿ࠀ࿿က쿿퀀퟿￿𐀀𿿿񀀀󿿿􀀀􏿽' .
+<http://a.example/s> <http://a.example/p> '€߿ࠀ࿿က쿿퀀퟿<EFBFBD>𐀀𿿽񀀀󿿽􀀀􏿽' .

tests/TurtleTests/LITERAL2_with_UTF8_boundaries.ttl

@@ -1 +1 @@
-<http://a.example/s> <http://a.example/p> "€߿ࠀ࿿က쿿퀀퟿￿𐀀𿿿񀀀󿿿􀀀􏿽" .
+<http://a.example/s> <http://a.example/p> "€߿ࠀ࿿က쿿퀀퟿<EFBFBD>𐀀𿿽񀀀󿿽􀀀􏿽" .

tests/TurtleTests/LITERAL_LONG1_with_UTF8_boundaries.ttl

@@ -1 +1 @@
-<http://a.example/s> <http://a.example/p> '''€߿ࠀ࿿က쿿퀀퟿￿𐀀𿿿񀀀󿿿􀀀􏿽''' .
+<http://a.example/s> <http://a.example/p> '''€߿ࠀ࿿က쿿퀀퟿<EFBFBD>𐀀𿿽񀀀󿿽􀀀􏿽''' .

tests/TurtleTests/LITERAL_LONG2_with_UTF8_boundaries.ttl

@@ -1 +1 @@
-<http://a.example/s> <http://a.example/p> """€߿ࠀ࿿က쿿퀀퟿￿𐀀𿿿񀀀󿿿􀀀􏿽""" .
+<http://a.example/s> <http://a.example/p> """€߿ࠀ࿿က쿿퀀퟿<EFBFBD>𐀀𿿽񀀀󿿽􀀀􏿽""" .

tests/TurtleTests/LITERAL_with_UTF8_boundaries.nt

@@ -0,0 +1 @@
+<http://a.example/s> <http://a.example/p> "\u0080\u07FF\u0800\u0FFF\u1000\uCFFF\uD000\uD7FF\uE000\uFFFD\U00010000\U0003FFFD\U00040000\U000FFFFD\U00100000\U0010FFFD" .

tests/TurtleTests/README

@@ -0,0 +1,65 @@
+This README is for the W3C RDF Working Group's Turtle test suite.
+This test suite contains four kinds of tests:
+
+  132 Evaluation (rdft:TestTurtleEval) - a pair of an input turtle
+  file and reference ntriples file.
+
+  77 Positive syntax (rdft:TestTurtlePositiveSyntax) - an input turtle
+  file with no syntax errors.
+
+  78 Negative syntax (rdft:TestTurtleNegativeSyntax) - an input turtle
+  file with at least one syntax error.
+
+  4 Negative Evaluation (rdft:TestTurtleNegativeEval) - a pair of an
+  input turtle file and reference ntriples file. These tests have the
+  same properties as rdft:TestTurtleNegativeSyntax.
+
+The manifest.ttl file in this directory lists all of the tests in the
+RDF WG's Turtle test suite. Each test is one of the above tests. All
+tests have a name (mf:name) and an input (mf:action). The Evaluation
+tests have an expected result (mf:result).
+
+• An implementation passes an Evaluation test if it parses the input
+  into a graph, parses the expecte result into another graph, and
+  those two graphs are isomorphic (see
+  <http://www.w3.org/TR/rdf11-concepts/#graph-isomorphism>).
+
+• An implementation passes a positive syntax test if it parses the
+  input.
+
+• An implementation passes a negative syntax test if it fails to parse
+  the input.
+
+
+RELATIVE IRI RESOLUTION:
+
+The home of the test suite is <http://www.w3.org/2013/TurtleTests/>.
+Per RFC 3986 section 5.1.3, the base IRI for parsing each file is the
+retrieval IRI for that file. For example, the tests turtle-subm-01 and
+turtle-subm-27 require relative IRI resolution against a base of
+<http://www.w3.org/2013/TurtleTests/turtle-subm-01.ttl> and
+<http://www.w3.org/2013/TurtleTests/turtle-subm-27.ttl> respectively.
+
+
+CHARACTER ENCODING:
+
+The Turtle language uses UTF-8 encoding. The following tests include
+non-ascii characters:
+  localName_with_assigned_nfc_bmp_PN_CHARS_BASE_character_boundaries
+  localName_with_assigned_nfc_PN_CHARS_BASE_character_boundaries *
+  localName_with_nfc_PN_CHARS_BASE_character_boundaries *
+  labeled_blank_node_with_PN_CHARS_BASE_character_boundaries *
+  LITERAL1_with_UTF8_boundaries *
+  LITERAL_LONG1_with_UTF8_boundaries *
+  LITERAL2_with_UTF8_boundaries *
+  LITERAL_LONG2_with_UTF8_boundaries *
+
+Those marked with a * include characters with codepoints greater than
+U+FFFD and are thus expressed as a pair of surrogate characters when
+represented in UCS2.
+
+
+See http://www.w3.org/2011/rdf-wg/wiki/Turtle_Test_Suite for more details.
+
+
+Eric Prud'hommeaux <eric+turtle@w3.org> - 11 June 2013.

tests/TurtleTests/SPARQL_style_base.ttl

@@ -0,0 +1,2 @@
+BASE <http://a.example/>
+<s> <http://a.example/p> <http://a.example/o> .

tests/TurtleTests/SPARQL_style_prefix.ttl

@@ -0,0 +1,2 @@
+PREFIX p: <http://a.example/>
+p:s <http://a.example/p> <http://a.example/o> .

tests/TurtleTests/comment_following_PNAME_NS.nt

@@ -0,0 +1 @@
+<http://a.example/s> <http://a.example/p> <http://a.example/> .

tests/TurtleTests/comment_following_PNAME_NS.ttl

@@ -0,0 +1,3 @@
+@prefix p: <http://a.example/> .
+<http://a.example/s> <http://a.example/p> p:#comment
+.

tests/TurtleTests/comment_following_localName.ttl

@@ -0,0 +1,3 @@
+@prefix p: <http://a.example/> .
+<http://a.example/s> <http://a.example/p> p:o#comment
+.

tests/TurtleTests/labeled_blank_node_with_PN_CHARS_BASE_character_boundaries.nt

@@ -0,0 +1 @@
+<http://a.example/s> <http://a.example/p> _:AZazÀÖØöø˿ͰͽͿ῿‌‍⁰↏Ⰰ⿯、퟿豈﷏ﷰ<EFBFBD>𐀀󯿽 .

tests/TurtleTests/labeled_blank_node_with_PN_CHARS_BASE_character_boundaries.ttl

@@ -0,0 +1 @@
+<http://a.example/s> <http://a.example/p> _:AZazÀÖØöø˿ͰͽͿ῿‌‍⁰↏Ⰰ⿯、퟿豈﷏ﷰ<EFBFBD>𐀀󯿽 .

tests/TurtleTests/labeled_blank_node_with_non_leading_extras.ttl

@@ -0,0 +1 @@
+<http://a.example/s> <http://a.example/p> _:a·̀ͯ‿.⁀ .