Unofficial Dub Package Documentation 
1 module more.common;
2 
3 import std.ascii;
4 import std.conv;
5 import std.range;
6 import std.stdio;
7 import std.string;
8 import std.bitmanip;
9 import std.format;
10 
11 import core.exception;
12 
13 import std.c.string : memmove;
14 
15 version(unittest)
16 {
17   import std.array;
18 }
19 
20 
21 void implement(string feature = "", string file = __FILE__, int line = __LINE__) {
22   string msg = "not implemented";
23   if(feature.length) {
24     msg = feature~' '~msg;
25   }
26   throw new Exception(msg, file, line);
27 }
28 
29 float stdTimeMillis(long stdTime)
30 {
31   return cast(float)stdTime / 10000f;
32 }
33 string prettyTime(float millis)
34 {
35   if (millis < 0) return "-"~prettyTime(-millis);
36     
37   if (millis < 1000)
38     return to!string(millis)~" millis";
39   if (millis < 60000)
40     return to!string(millis / 1000)~" seconds";
41   if (millis < 3600000)
42     return to!string(millis / 60000)~" minutes";
43   if (millis < 86400000)
44     return to!string(millis / 3600000)~" hours";
45   return to!string(millis / 86400000)~" days";
46 }
47 
48 template isChar(T) {
49   static if(is(T == char) ||
50 	    is(T == const char) ||
51 	    is(T == immutable char))
52     enum isChar = true;
53   else
54     enum isChar = false;
55 }
56 
57 template ArrayElementType(Type)
58 {
59   static if(is(Type : E[], E))
60     alias ArrayElementType = E;
61   else
62     alias ArrayElementType = void;
63 }
64 template TypeTupleStrings(alias sep, T...)
65 {
66   static if(T.length == 0) {
67     immutable string TypeTupleStrings = "";
68   } else static if(T.length == 1) {
69     immutable string TypeTupleStrings = T[0].stringof;
70   } else {
71     immutable string TypeTupleStrings = T[0].stringof ~ sep ~ TypeTupleStrings!(sep, T[1..$]);
72   }
73 }
74 template TupleRange(int from, int to) if (from <= to) {
75   static if (from >= to) {
76     alias TupleRange = TypeTuple!();
77   } else {
78     alias TupleRange = TypeTuple!(from, TupleRange!(from + 1, to));
79   }
80 }
81 
82 public interface IDisposable
83 {
84   void dispose();
85 }
86 
87 
88 enum outerBar = "=========================================";
89 enum innerBar = "-----------------------------------------";
90 void startTest(string name)
91 {
92   writeln(outerBar);
93   writeln(name, ": Start");
94   writeln(innerBar);
95 }
96 void endFailedTest(string name)
97 {
98   writeln(innerBar);
99   writeln(name, ": Failed");
100   writeln(outerBar);
101 }
102 void endPassedTest(string name)
103 {
104   writeln(innerBar);
105   writeln(name, ": Passed");
106   writeln(outerBar);
107 }
108 template scopedTest(string name) {
109   enum scopedTest =
110     "startTest(\""~name~"\");"~
111     "scope(failure) {stdout.flush();endFailedTest(\""~name~"\");}"~
112     "scope(success) endPassedTest(\""~name~"\");";
113 }
114 void writeSection(string name)
115 {
116   writeln(innerBar);
117   writeln(name);
118   writeln(innerBar);
119 }
120 void assertEqual(string expected, string actual) pure
121 {
122   if(expected != actual) {
123     throw new AssertError(format("Expected %s Actual %s",
124 				 expected ? ('"' ~ expected ~ '"') : "<null>",
125 				 actual   ? ('"' ~ actual   ~ '"') : "<null>"));
126   }
127 }
128 
129 template Unroll(alias CODE, alias N, alias SEP="")
130 {
131     enum NEW_CODE = replace(CODE, "%", "%1$d");
132     enum Unroll = iota(N).map!(i => format(NEW_CODE, i)).join(SEP);
133 }
134 template UnrollTuple(alias CODE, alias SEP, T...)
135 {
136     enum NEW_CODE = replace(CODE, "%", "%1$d");
137     enum UnrollTuple = T.map!(t => format(NEW_CODE, t)).join(SEP);
138 }
139 
140 
141 public alias DataHandler = void delegate(ubyte[] data);
142 public alias StringHandler = void delegate(string data);
143 public interface IDataHandler : IDisposable
144 {
145   void handleData(ubyte[] data);
146 }
147 
148 string debugChar(char c)
149 {
150   switch(c) {
151   case '\r':return r"'\r'";
152   case '\n':return r"'\n'";
153   default:
154     if(isPrintable(c)) return "'" ~ c ~ "'";
155     return to!string(cast(uint)c);
156   }
157 }
158 
159 void dcpy(void* destination, const ubyte[] source) pure nothrow
160 {
161   (cast(ubyte*)destination)[0..source.length][] = source;
162 }
163 void dcpy(void* destination, const char[] source) pure nothrow
164 {
165   (cast(char*)destination)[0..source.length][] = source;
166 }
167 /+
168 void dcpy(Source)(void * destination, Source source) pure nothrow
169 {
170   (cast(ubyte*)destination)[0..source.length][] = source;
171 }
172 +/
173 
174 /+
175 void trimNewline(ref inout(char)[] line) {
176   // remove ending characters
177   while(true) {
178     if( line.length <= 0 || (line[$-1] != '\n' && line[$-1] != '\r') ) return;
179     line = line[0..$-1];
180   }
181 }
182 +/
183 
184 void trimNewline(inout(char)[]* line) {
185   // remove ending characters
186   while(true) {
187     if( (*line).length <= 0 || ((*line)[$-1] != '\n' && (*line)[$-1] != '\r') ) return;
188     (*line) = (*line)[0..$-1];
189   }
190 }
191 
192 
193 version(unittest_common) unittest
194 {
195   mixin(scopedTest!("trimNewline"));
196 
197   void testTrimNewline(string s, string expected) {
198     //writef("'%s' => ", escape(s));
199     trimNewline(&s);
200     //writefln("'%s'", escape(s));
201     assert(expected == s);
202   }
203 
204   testTrimNewline("", "");
205   testTrimNewline("\r", "");
206   testTrimNewline("\n", "");
207   testTrimNewline("1234", "1234");
208   testTrimNewline("abcd  \n", "abcd  ");
209   testTrimNewline("hello\r\r\r\n\n\r\r\n", "hello");
210 
211 }
212 
213 
214 void readFullSize(File file, char[] buffer) {
215   while(true) {
216     char[] lastRead = file.rawRead(buffer);
217     if(lastRead.length == buffer.length) return;
218     if(lastRead.length == 0) throw new Exception("File did not have enough data left to fill the buffer");
219     buffer = buffer[lastRead.length..$];
220   }
221 }
222 
223 // returns true on success, false if the file reached EOF
224 bool tryReadFullSize(File file, char[] buffer) {
225   while(true) {
226     char[] lastRead = file.rawRead(buffer);
227     if(lastRead.length == buffer.length) return true;
228     if(lastRead.length == 0) return false;
229     buffer = buffer[lastRead.length..$];
230   }
231 }
232 
233 alias ubyte[] function(ubyte[] oldBuffer, size_t newLength, bool copy) Allocator;
234 alias void function(ubyte[] buffer) Deallocator;
235 
236 ubyte[] defaultAllocator(ubyte[] oldBuffer, size_t newLength, bool copy)
237 {
238   oldBuffer.length = newLength;
239   return oldBuffer;
240 }
241 void defaultDeallocator(ubyte[] buffer)
242 {
243   // do nothing, the garbage collector handles it
244 }
245 alias CustomLineParser!(defaultAllocator, defaultDeallocator) LineParser;
246 
247 template CustomLineParser(A...)
248 {
249 struct CustomLineParser
250 {
251   size_t expandLength;
252 
253   ubyte[] buffer;
254 
255   ubyte *lineStart;       // Points to the start of the next line
256   size_t lineCheckOffset; // Offset from lineStart to the next character to check
257   size_t lineDataLimit;   // Length from lineStart to the end of the data
258 
259   public this(ubyte[] buffer, size_t expandLength = 256)
260   {
261     this.expandLength = expandLength;
262 
263     this.buffer = buffer;
264     this.lineStart = buffer.ptr;
265     this.lineCheckOffset = 0;
266     this.lineDataLimit = 0;
267   }
268   @property public void put(Data)(Data data) if ( (*data.ptr).sizeof == 1 )
269   {
270     debug writefln("[debug] ---> put(%4d bytes) : start %d check %d limit %d",
271 		   data.length, (lineStart - buffer.ptr), lineCheckOffset, lineDataLimit);
272 
273     size_t newDataOffset = (lineStart - buffer.ptr) + lineDataLimit;
274     size_t bytesLeft = buffer.length - newDataOffset;
275 
276     if(bytesLeft < data.length) {
277       size_t lineStartOffset = lineStart - buffer.ptr;
278       size_t defaultExpandLength = this.buffer.length + this.expandLength;
279       size_t neededLength = newDataOffset + data.length;
280 
281       //this.buffer.length = (neededLength >= defaultExpandLength) ? neededLength : defaultExpandLength;
282       ubyte[] newBuffer = A[0](this.buffer, (neededLength >= defaultExpandLength) ? neededLength : defaultExpandLength, true);
283       A[1](this.buffer);
284       this.buffer = newBuffer;
285 
286       lineStart = this.buffer.ptr + lineStartOffset;
287     }
288 
289     dcpy(lineStart + lineDataLimit, data);
290     lineDataLimit += data.length;
291 
292     debug writefln("[debug] <--- put             : start %d check %d limit %d",
293 		   (lineStart - buffer.ptr), lineCheckOffset, lineDataLimit);
294   }
295 
296   /// Returns null when it has parsed all the lines that have been added
297   public ubyte[] getLine()
298   {
299     debug writefln("[debug] ---> getLine         : start %d check %d limit %d",
300 		   (lineStart - buffer.ptr), lineCheckOffset, lineDataLimit);
301 
302     while(lineCheckOffset < lineDataLimit) {
303       debug {
304 	char c = lineStart[lineCheckOffset];
305 	writefln("[debug] start[%s] = %s", lineCheckOffset, debugChar(c));
306       }
307       if(lineStart[lineCheckOffset] == '\n') {
308 	ubyte[] line;
309 	if(lineCheckOffset > 0 && lineStart[lineCheckOffset - 1] == '\r') {
310 	  line = lineStart[0..lineCheckOffset - 1];
311 	} else {
312 	  line = lineStart[0..lineCheckOffset];
313 	}
314 	lineCheckOffset++;
315 	lineDataLimit -= lineCheckOffset;
316 	if(lineDataLimit > 0) {
317 	  lineStart = lineStart + lineCheckOffset;
318 	} else {
319 	  lineStart = buffer.ptr;
320 	}
321 	lineCheckOffset = 0;
322 
323 	debug writefln("[debug] <--- getLine line    : start %d check %d limit %d",
324 		       (lineStart - buffer.ptr), lineCheckOffset, lineDataLimit);
325 	return line;
326       }
327       lineCheckOffset++;
328     }
329 
330     if(lineStart == buffer.ptr) {
331       debug writeln("[debug] No data");
332       return null;
333     }
334 
335     assert(lineDataLimit > 0);
336 
337     //
338     // Move remaining data to the beginning of the buffer
339     //
340     debug writeln("[debug] Moving line data to beginning of buffer");
341     memmove(buffer.ptr, lineStart, lineDataLimit);
342     lineStart = buffer.ptr;
343     debug {
344       foreach(i,c; lineStart[0..lineDataLimit]) {
345 	writefln("[debug] line[%s] = %s", i, debugChar(c));
346       }
347     }
348     debug writefln("[debug] <--- getLine no-line : start %d check %d limit %d",
349 		   (lineStart - buffer.ptr), lineCheckOffset, lineDataLimit);
350     return null;
351   }
352 }
353 
354 }
355 version(unittest_common) unittest
356 {
357   mixin(scopedTest!("LineParser"));
358 
359   void TestLineParser(LineParser lineParser)
360   {
361     lineParser.put("\n");
362     assertEqual("", cast(string)lineParser.getLine());
363     assert(!lineParser.getLine());
364 
365     lineParser.put("\r\n");
366     assertEqual("", cast(string)lineParser.getLine());
367     assert(!lineParser.getLine());
368 
369     lineParser.put("\r\r\n");
370     assertEqual("\r", cast(string)lineParser.getLine());
371     assert(!lineParser.getLine());
372 
373     lineParser.put("\n\n\n");
374     assertEqual("", cast(string)lineParser.getLine());
375     assertEqual("", cast(string)lineParser.getLine());
376     assertEqual("", cast(string)lineParser.getLine());
377     assert(!lineParser.getLine());
378 
379     lineParser.put("\r\r\r\n\r\n\n\r\r\r\n");
380     assertEqual("\r\r", cast(string)lineParser.getLine());
381     assertEqual("", cast(string)lineParser.getLine());
382     assertEqual("", cast(string)lineParser.getLine());
383     assertEqual("\r\r", cast(string)lineParser.getLine());
384     assert(!lineParser.getLine());
385 
386     lineParser.put("abcd\n");
387     assertEqual("abcd", cast(string)lineParser.getLine());
388     assert(!lineParser.getLine());
389 
390     lineParser.put("abcd\r\n");
391     assertEqual("abcd", cast(string)lineParser.getLine());
392     assert(!lineParser.getLine());
393 
394     lineParser.put("abcd\nefgh\r\n");
395     assertEqual("abcd", cast(string)lineParser.getLine());
396     assertEqual("efgh", cast(string)lineParser.getLine());
397     assert(!lineParser.getLine());
398 
399     lineParser.put("abcd\r\nefghijkl");
400     assertEqual("abcd", cast(string)lineParser.getLine());
401     assert(!lineParser.getLine());
402     lineParser.put("\n");
403     assertEqual("efghijkl", cast(string)lineParser.getLine());
404     assert(!lineParser.getLine());
405 
406     lineParser.put("abcd\n");
407     lineParser.put("abcd\r\n");
408     lineParser.put("abcd\n");
409     assertEqual("abcd", cast(string)lineParser.getLine());
410     assertEqual("abcd", cast(string)lineParser.getLine());
411     assertEqual("abcd", cast(string)lineParser.getLine());
412     assert(!lineParser.getLine());
413 
414     lineParser.put("a");
415     assert(!lineParser.getLine());
416     lineParser.put("bc");
417     assert(!lineParser.getLine());
418     lineParser.put("d");
419     assert(!lineParser.getLine());
420     lineParser.put("\r\ntu");
421     lineParser.put("v");
422     assertEqual("abcd", cast(string)lineParser.getLine());
423     assert(!lineParser.getLine());
424     lineParser.put("\r");
425     assert(!lineParser.getLine());
426     lineParser.put("\n");
427     assertEqual("tuv", cast(string)lineParser.getLine());
428     assert(!lineParser.getLine());
429   }
430 
431   TestLineParser(LineParser([],1));
432   TestLineParser(LineParser([0,0,0], 3));
433   TestLineParser(LineParser(new ubyte[256]));
434 }
435 
436 
437 enum noEndingQuoteMessage = "found '\"' with no ending '\"'";
438 
439 /**
440  * Used to parse the fields in <i>line</i> to the <i>fields</i> sink.
441  * line is a single line without the line ending character.
442  * returns error message on error
443  */
444 string tryParseFields(string comment = "#", T, C)(T fields, C[] line) if(isOutputRange!(T,C))
445 {
446   size_t off = 0;
447   size_t startOfField;
448 
449   Unqual!C c;
450 
451   while(true) {
452 
453     // Skip whitespace
454     while(true) {
455       if(off >= line.length) return null;
456       c = line[off];
457       if(c != ' ' && c != '\t') break;
458       off++;
459     }
460 
461     // check for comment
462     static if(comment != null && comment.length == 1) {
463 
464       if(c == comment[0]) return null;
465 
466     } else static if(comment != null && comment.length > 1) {
467 
468       if(c == comment[0]) {
469 	size_t i = 1;
470 	while(true) {
471 	  if(off + i > line.length) break;
472 	  c = line[off + i];
473 	  if(c != comment[i]) break;
474 
475 	  i++;
476 	  if(i >= comment.length) return null;
477 	}
478 
479 	c = comment[0]; // restore state
480       }
481 
482     }
483 
484     if(c == '"') {
485       off++;
486       startOfField = off;
487       while(true) {
488 	if(off >= line.length) return noEndingQuoteMessage;
489 	c = line[off];
490 	if(c == '"') {
491 	  fields.put(line[startOfField..off]);
492 	  break;
493 	}
494 	if(c == '\\') {
495 	  throw new Exception("Escaping string not implemented yet");
496 	}
497 	off++;
498       }
499     } else {
500       startOfField = off;
501       while(true) {
502 	off++;
503 	if(off >= line.length) {
504 	  fields.put(line[startOfField..$]);
505 	  return null;
506 	}
507 	c = line[off];
508 	if(c == ' ' || c == '\t') {
509 	  fields.put(line[startOfField..off]);
510 	  break;
511 	}
512       }
513     }
514   }
515 }
516 
517 version(unittest_common) unittest
518 {
519   mixin(scopedTest!("tryParseFields"));
520 
521   string line;
522   auto fields = appender!(string[])();
523 
524   line = "\"Missing ending quote";
525   assert(tryParseFields(fields, line) == noEndingQuoteMessage);
526 
527   line = "\"Missing ending quote\n";
528   assert(tryParseFields(fields, line) == noEndingQuoteMessage);
529 
530   line = "\"Missing ending quote\nOn next line \"";
531   assert(tryParseFields(fields, line) == noEndingQuoteMessage);
532 
533   line = "";
534   assert(tryParseFields(fields, line) == null);
535 }
536 
537 
538 void bigEndianSetUshort(ubyte* bytes, ushort value)
539 {
540   *bytes       = cast(ubyte)(value >> 8);
541   *(bytes + 1) = cast(ubyte)(value     );
542 }
543 struct ArrayList(V)
544 {
545   public V[] array;
546   public size_t count;
547 
548   this(size_t initialSize)
549   {
550     if(initialSize < 1) throw new Exception("Cannot give an initial size of 0");
551     array = new V[initialSize];
552     count = 0;
553   }
554   this(Range)(Range r)
555   {
556     array = new V[r.length];
557     foreach(i, element; r) {
558       array[i] = element;
559     }
560     count = r.length;
561   }
562   void clear()
563   {
564     count = 0;
565   }
566   void checkSizeBeforeAdd()
567   {
568     if(count >= array.length) {
569       array.length *= 2;
570     }
571   }
572   public void put(V element)
573   {
574     checkSizeBeforeAdd();
575     array[count++] = element;
576   }
577   public void forwardTo(Sink)(Sink sink)
578   {
579     foreach(i; 0..count) {
580       sink.add(array[i]);
581     }
582   }
583   public void removeAt(size_t index)
584   {
585     for(auto i = index; i < count - 1; i++) {
586       array[index] = array[index + 1];
587     }
588     count--;
589   }
590 }
591 
592 
593 immutable string[] escapeTable =
594   [
595    "\\0"  ,
596    "\\x01",
597    "\\x02",
598    "\\x03",
599    "\\x04",
600    "\\x05",
601    "\\x06",
602    "\\a",  // Bell
603    "\\b",  // Backspace
604    "\\t",
605    "\\n",
606    "\\v",  // Vertical tab
607    "\\f",  // Form feed
608    "\\r",
609    "\\x0E",
610    "\\x0F",
611    "\\x10",
612    "\\x11",
613    "\\x12",
614    "\\x13",
615    "\\x14",
616    "\\x15",
617    "\\x16",
618    "\\x17",
619    "\\x18",
620    "\\x19",
621    "\\x1A",
622    "\\x1B",
623    "\\x1C",
624    "\\x1D",
625    "\\x1E",
626    "\\x1F",
627    " ", //
628    "!", //
629    "\"", //
630    "#", //
631    "$", //
632    "%", //
633    "&", //
634    "'", //
635    "(", //
636    ")", //
637    "*", //
638    "+", //
639    ",", //
640    "-", //
641    ".", //
642    "/", //
643    "0", //
644    "1", //
645    "2", //
646    "3", //
647    "4", //
648    "5", //
649    "6", //
650    "7", //
651    "8", //
652    "9", //
653    ":", //
654    ";", //
655    "<", //
656    "=", //
657    ">", //
658    "?", //
659    "@", //
660    "A", //
661    "B", //
662    "C", //
663    "D", //
664    "E", //
665    "F", //
666    "G", //
667    "H", //
668    "I", //
669    "J", //
670    "K", //
671    "L", //
672    "M", //
673    "N", //
674    "O", //
675    "P", //
676    "Q", //
677    "R", //
678    "S", //
679    "T", //
680    "U", //
681    "V", //
682    "W", //
683    "X", //
684    "Y", //
685    "Z", //
686    "[", //
687    "\\", //
688    "]", //
689    "^", //
690    "_", //
691    "`", //
692    "a", //
693    "b", //
694    "c", //
695    "d", //
696    "e", //
697    "f", //
698    "g", //
699    "h", //
700    "i", //
701    "j", //
702    "k", //
703    "l", //
704    "m", //
705    "n", //
706    "o", //
707    "p", //
708    "q", //
709    "r", //
710    "s", //
711    "t", //
712    "u", //
713    "v", //
714    "w", //
715    "x", //
716    "y", //
717    "z", //
718    "{", //
719    "|", //
720    "}", //
721    "~", //
722    "\x7F", //
723    "\x80", "\x81", "\x82", "\x83", "\x84", "\x85", "\x86", "\x87", "\x88", "\x89", "\x8A", "\x8B", "\x8C", "\x8D", "\x8E", "\x8F",
724    "\x90", "\x91", "\x92", "\x93", "\x94", "\x95", "\x96", "\x97", "\x98", "\x99", "\x9A", "\x9B", "\x9C", "\x9D", "\x9E", "\x9F",
725    "\xA0", "\xA1", "\xA2", "\xA3", "\xA4", "\xA5", "\xA6", "\xA7", "\xA8", "\xA9", "\xAA", "\xAB", "\xAC", "\xAD", "\xAE", "\xAF",
726    "\xB0", "\xB1", "\xB2", "\xB3", "\xB4", "\xB5", "\xB6", "\xB7", "\xB8", "\xB9", "\xBA", "\xBB", "\xBC", "\xBD", "\xBE", "\xBF",
727    "\xC0", "\xC1", "\xC2", "\xC3", "\xC4", "\xC5", "\xC6", "\xC7", "\xC8", "\xC9", "\xCA", "\xCB", "\xCC", "\xCD", "\xCE", "\xCF",
728    "\xD0", "\xD1", "\xD2", "\xD3", "\xD4", "\xD5", "\xD6", "\xD7", "\xD8", "\xD9", "\xDA", "\xDB", "\xDC", "\xDD", "\xDE", "\xDF",
729    "\xE0", "\xE1", "\xE2", "\xE3", "\xE4", "\xE5", "\xE6", "\xE7", "\xE8", "\xE9", "\xEA", "\xEB", "\xEC", "\xED", "\xEE", "\xEF",
730    "\xF0", "\xF1", "\xF2", "\xF3", "\xF4", "\xF5", "\xF6", "\xF7", "\xF8", "\xF9", "\xFA", "\xFB", "\xFC", "\xFD", "\xFE", "\xFF",
731    ];
732 string escape(char c) {
733   return escapeTable[c];
734 }
735 string escape(dchar c) {
736   return (c < escapeTable.length) ? escapeTable[c] : to!string(c);
737 }
738 inout(char)[] escape(inout(char)[] str) pure {
739   size_t extra = 0;
740   foreach(c; str) {
741     if(c == '\r' || c == '\t' || c == '\n' || c == '\\') {
742       extra++;
743     }
744   }
745 
746   if(extra == 0) return str;
747 
748   char[] newString = new char[str.length + extra];
749   size_t oldIndex, newIndex = 0;
750   for(oldIndex = 0; oldIndex < str.length; oldIndex++) {
751     auto c = str[oldIndex];
752     switch(c) {
753     case '\r':
754       newString[newIndex++] = '\\';
755       newString[newIndex++] = 'r';
756       break;
757     case '\t':
758       newString[newIndex++] = '\\';
759       newString[newIndex++] = 't';
760       break;
761     case '\n':
762       newString[newIndex++] = '\\';
763       newString[newIndex++] = 'n';
764       break;
765     case '\\':
766       newString[newIndex++] = '\\';
767       newString[newIndex++] = '\\';
768       break;
769     default:
770       newString[newIndex++] = c;
771     }
772   }
773 
774   assert(newIndex == newString.length);
775 
776   return cast(inout(char)[])newString;
777 }
778 version(unittest_common) unittest
779 {
780   mixin(scopedTest!("escape"));
781 
782   assert (char.max == escapeTable.length - 1);
783 
784   for(auto c = ' '; c <= '~'; c++) {
785     assert(c == escape(c)[0]);
786   }
787   assert("\\0" == escape(0));
788   assert("\\r" == escape('\r'));
789   assert("\\n" == escape('\n'));
790   assert("\\t" == escape('\t'));
791 
792 
793   assert("\\r" == escape("\r"));
794   assert("\\t" == escape("\t"));
795   assert("\\n" == escape("\n"));
796   assert("\\\\" == escape("\\"));
797 }
798 
799 
800 struct StdoutWriter
801 {
802   void put(const (char)[] str) {
803     write(str);
804   }
805 }
806 
807 struct ReadBuffer(T)
808 {
809   const(T)* next;
810   const T* limit;
811 }
812 struct WriteBuffer(T)
813 {
814   T* next;
815   const T* limit;
816   this(T[] array) {
817     this.next = array.ptr;
818     this.limit = array.ptr + array.length;
819   }
820   T[] slice(T* start) {
821     return start[0..next-start];
822   }
823   void put(const(T)[] array) {
824     if(next + array.length > limit)
825       throw new Exception(format("buffer has %s elements left but you tried to add %s",
826 				 limit - next, array.length));
827 
828     foreach(value; array) {
829       *next = value;
830       next++;
831     }
832   }
833 }
834 
835 alias void delegate(const(char)[] msg) const Writer;
836 
837 alias size_t delegate(char[] buffer) CharReader;
838 alias size_t delegate(ubyte[] buffer) DataReader;
839 
840 struct FileCharReader
841 {
842   File file;
843   size_t read(char[] buffer) {
844     return file.rawRead(buffer).length;
845   }
846 }
847 
848 struct AsciiBufferedInput
849 {
850   CharReader reader;
851   char[] buffer;
852   size_t start;
853   size_t limit;
854 
855   this(inout(char)[] s) {
856     this.reader = &emptyRead;
857     this.buffer = new char[s.length + 1];
858     this.buffer[0..$-1] = s;
859     this.start = 0;
860     this.limit = s.length;
861   }
862   private size_t emptyRead(char[] buffer) { return 0; }
863 
864   char[] sliceSaved() {
865     return buffer[start..limit];
866   }
867 
868   void clear() {
869     start = 0;
870     limit = 0;
871   }
872 
873   // returns true if there is data, false on EOF
874   bool readNoSave() {
875     start = 0;
876     limit = reader(buffer);
877     return limit > 0;
878   }
879 
880   // returns true if read succeeded, false on EOF
881   bool read(size_t* offsetToShift) {
882     //writefln("[DEBUG] --> read(start=%s, limit=%s)", start, limit);
883     size_t leftover = limit - start;
884 
885     if(leftover) {
886       if(leftover >= buffer.length) {
887 	throw new Exception("Buffer not large enough");
888       }
889 
890       if(start > 0) {
891 	memmove(buffer.ptr, buffer.ptr + start, leftover);
892 	*offsetToShift -= leftover;
893       }
894     }
895 
896     start = 0;
897     limit = leftover;
898     size_t readLength = reader(buffer[leftover..$]);
899     if(readLength == 0) {
900       //writefln("[DEBUG] <-- read(start=%s, length=%s) = false", start, length);
901       return false;
902     }
903 
904     limit += readLength;
905     return true;
906   }
907 
908 
909 }
910 
911 
912 struct FormattedBinaryWriter
913 {
914   scope void delegate(const(char)[]) sink;
915 
916   ubyte[] columnBuffer;
917 
918   string offsetFormat;
919   mixin(bitfields!
920 	(bool, "hex", 1,
921 	 bool, "text", 1,
922 	 void, "", 6));
923 
924   size_t cachedData;
925 
926   uint offset;
927 
928   this(scope void delegate(const(char)[]) sink, ubyte[] columnBuffer,
929        ubyte offsetTextWidth = 8, bool hex = true, bool text = true) {
930     this.sink = sink;
931     this.columnBuffer = columnBuffer;
932 
933     if(offsetTextWidth == 0) {
934       offsetFormat = null;
935     } else {
936       offsetFormat = "%0"~to!string(offsetTextWidth)~"x";
937     }
938 
939     this.hex = hex;
940     this.text = text;
941   }
942 
943   void writeAscii(byte b) {
944     if(b <= '~' && b >= ' ') {
945       sink((cast(char*)&b)[0..1]);
946     } else {
947       sink(".");
948     }
949   }
950 
951   void writeRow(T)(T data) {
952     bool atFirst = true;
953     void prefix() {
954       if(atFirst) {atFirst = false; }
955       else { sink(" "); }
956     }
957     if(offsetFormat) {
958       prefix();
959       formattedWrite(sink, offsetFormat, offset);
960     }
961 
962     if(hex) {
963       prefix();
964       foreach(b; data) {
965 	formattedWrite(sink, " %02x", b);
966       }
967     }
968 
969     if(text) {
970       prefix();
971       foreach(b; data) {
972 	writeAscii(b);
973       }
974     }
975 
976     sink("\n");
977     offset += columnBuffer.length;
978   }
979   void finish() {
980     if(cachedData > 0) {
981       bool atFirst = true;
982       void prefix() {
983 	if(atFirst) {atFirst = false; }
984 	else { sink(" "); }
985       }
986       if(offsetFormat) {
987 	prefix();
988 	formattedWrite(sink, offsetFormat, offset);
989       }
990 
991       if(hex) {
992 	prefix();
993 	foreach(b; columnBuffer[0..cachedData]) {
994 	  formattedWrite(sink, " %02x", b);
995 	}
996 	foreach(b; cachedData..columnBuffer.length) {
997 	  sink("   ");
998 	}
999       }
1000 
1001       if(text) {
1002 	prefix();
1003 	foreach(b; columnBuffer[0..cachedData]) {
1004 	  writeAscii(b);
1005 	}
1006       }
1007 
1008       sink("\n");
1009       offset += columnBuffer.length;
1010 
1011       cachedData = 0;
1012     }
1013   }
1014   void put(scope ubyte[] data) in { assert(data.length); } body {
1015     if(cachedData > 0) {
1016       auto combineLength = columnBuffer.length - cachedData;
1017       if(combineLength > data.length) {
1018 	combineLength = data.length;
1019       }
1020       columnBuffer[cachedData..cachedData+combineLength] = data[];
1021       writeRow(columnBuffer);
1022       data = data[combineLength..$];
1023     }
1024 
1025     while(data.length >= columnBuffer.length) {
1026       writeRow(data[0..columnBuffer.length]);
1027       data = data[columnBuffer.length..$];
1028     }
1029       
1030     if(data.length > 0) {
1031       columnBuffer[0..data.length][] = data;
1032       cachedData = data.length;
1033     }
1034   }
1035 }
1036 
1037 
1038 
1039 //
1040 // Range Initializers
1041 //
1042 string arrayRange(char min, char max, string initializer) {
1043   string initializers = "";
1044   for(char c = min; c < max; c++) {
1045     initializers ~= "'"~c~"': "~initializer~",\n";
1046   }
1047   initializers ~= "'"~max~"': "~initializer;
1048   return initializers;
1049 }
1050 string rangeInitializers(string[] s...) {
1051   if(s.length % 2 != 0) assert(0, "must supply an even number of arguments to rangeInitializers");
1052   string code = "["~rangeInitializersCurrent(s);
1053   //assert(0, code); // uncomment to see the code
1054   return code;
1055 }
1056 string rangeInitializersCurrent(string[] s) {
1057   string range = s[0];
1058   if(range[0] == '\'') {
1059     if(range.length == 3 || (range.length == 4 && range[1] == '\\')) {
1060       if(range[$-1] != '\'') throw new Exception(format("a single-character range %s started with an apostrophe (') but did not end with one", range));
1061       return range ~ ":" ~ s[1] ~ rangeInitializersNext(s);
1062     }
1063   } else {
1064     throw new Exception(format("range '%s' not supported", range));
1065   }
1066   char min = range[1];
1067   char max = range[5];
1068   return arrayRange(min, max, s[1]) ~ rangeInitializersNext(s);
1069 }
1070 string rangeInitializersNext(string[] s...) {
1071   if(s.length <= 2) return "]";
1072   return ",\n"~rangeInitializersCurrent(s[2..$]);
1073 }
1074 
1075 
1076 
1077 struct StringByLine
1078 {
1079   string s;
1080   size_t startOfLineOffset;
1081   size_t endOfLineOffset;
1082   this(string s) @safe pure nothrow @nogc
1083   {
1084     this.s = s;
1085     this.startOfLineOffset = 0;
1086     this.endOfLineOffset = 0;
1087     popFront;
1088   }
1089   @property bool empty() pure nothrow @safe @nogc
1090   {
1091     return startOfLineOffset >= s.length;
1092   }
1093   @property auto front() pure nothrow @safe @nogc
1094   {
1095     return s[startOfLineOffset..endOfLineOffset];
1096   }
1097   @property void popFront() pure nothrow @safe @nogc
1098   {
1099     if(startOfLineOffset < s.length) {
1100       startOfLineOffset = endOfLineOffset;
1101       while(true) {
1102 	if(endOfLineOffset >= s.length) break;
1103 	if(s[endOfLineOffset] == '\n') {
1104 	  endOfLineOffset++;
1105 	  break;
1106 	}
1107 	endOfLineOffset++;
1108       }
1109     }
1110   }
1111 }
1112 auto byLine(string s) pure nothrow @safe @nogc {
1113   return StringByLine(s);
1114 }
1115 version(unittest_common) unittest
1116 {
1117   mixin(scopedTest!("StringByLine"));
1118   
1119   void test(string[] expectedStrings, string s, size_t testLine = __LINE__)
1120   {
1121     auto stringByLine = s.byLine();
1122 
1123     foreach(expectedString; expectedStrings) {
1124       if(stringByLine.empty) {
1125 	writefln("Expected string '%s' but no more strings", escape(expectedString));
1126 	assert(0);
1127       }
1128       if(expectedString != stringByLine.front) {
1129 	writefln("Expected: '%s'", escape(expectedString));
1130 	writefln("Actual  : '%s'", escape(stringByLine.front));
1131 	assert(0);
1132       }
1133       stringByLine.popFront;
1134     }
1135 
1136     if(!stringByLine.empty) {
1137       writefln("Expected no more strings but got another '%s'", escape(stringByLine.front));
1138       assert(0);
1139     }
1140   }
1141 
1142   test([], "");
1143   test(["a"], "a");
1144   test(["a\n"], "a\n");
1145   test(["abc"], "abc");
1146   test(["abc\n"], "abc\n");
1147 
1148   test(["abc\n", "123"], "abc\n123");
1149   test(["abc\n", "123\n"], "abc\n123\n");
1150 
1151 }
1152 
1153 
1154 
1155 enum BufferTooSmall
1156 {
1157   returnPartialData,
1158   throwException,
1159   resizeBuffer,
1160 }
1161 
1162 /**
1163    The LinesChunker reads as many lines as it can.  If the buffer runs out in the
1164    middle of a line it will return all the previous full lines.  On the next
1165    read it will move the left over data to the beginning of the buffer and continue reading.
1166    If it cannot read a full line it will either return partial lines or it will throw an
1167    error depending on what the user specifies.
1168    Options on how to handle lines that are too long
1169      1. Return partial lines
1170      2. Resize the buffer to hold the entire line
1171      3. Throw an exception/cause an error
1172 */
1173 struct LinesChunker
1174 {
1175   char[] buffer;
1176   BufferTooSmall tooSmall;
1177   private char[] leftOver;
1178   
1179   this(char[] buffer, BufferTooSmall tooSmall) {
1180     this.buffer = buffer;
1181     this.tooSmall = tooSmall;
1182   }
1183   size_t read(CharReader reader)
1184   {
1185     //
1186     // Handle leftOver Data
1187     //
1188     size_t bufferOffset;
1189     if(leftOver.length == 0) {
1190       bufferOffset = 0;
1191     } else {
1192       // TODO: do I need this check? Can this ever happen?
1193       if(leftOver.ptr != buffer.ptr) {
1194 	memmove(buffer.ptr, leftOver.ptr, leftOver.length);
1195       }
1196       bufferOffset = leftOver.length;
1197       leftOver = null;
1198     }
1199 
1200     //
1201     // Read More Data
1202     //
1203     while(true) {
1204       if(bufferOffset >= buffer.length) {
1205 	if(tooSmall == BufferTooSmall.returnPartialData) {
1206 	  return bufferOffset;
1207 	} else if(tooSmall == BufferTooSmall.resizeBuffer) {
1208 	  throw new Exception("BufferTooSmall.resizeBuffer is not implemented in LinesChunker");
1209 	}
1210 	throw new Exception(format("the current buffer of length %s is too small to hold the current line", buffer.length));
1211       }
1212 
1213       size_t readLength = reader(buffer[bufferOffset .. $]);
1214       if(readLength == 0) return bufferOffset;
1215 
1216       auto totalLength = bufferOffset + readLength;
1217       auto i = totalLength - 1;
1218       while(true) {
1219 	auto c = buffer[i];
1220 	if(c == '\n') {
1221 	  leftOver = buffer[i+1..totalLength];
1222 	  return i+1;
1223 	}
1224 	if(i == bufferOffset) {
1225 	  break;
1226 	}
1227 	i--;
1228       }
1229 
1230       bufferOffset = totalLength;
1231     }
1232   }
1233 }
1234 
1235 version(unittest_common)
1236 {
1237   struct CustomChunks {
1238     string[] chunks;
1239     size_t chunkIndex;
1240     size_t read(char[] buffer) {
1241       if(chunkIndex >= chunks.length) return 0;
1242       auto chunk = chunks[chunkIndex++];
1243       if(chunk.length > buffer.length) {
1244 	assert(0, format("Chunk at index %s is %s bytes but the buffer is only %s", chunkIndex, chunk.length, buffer.length));
1245       }
1246       buffer[0..chunk.length] = chunk;
1247       return chunk.length;
1248     }
1249   }
1250 }
1251 struct LinesReader
1252 {
1253   CharReader reader;
1254   LinesChunker chunker;
1255   size_t currentBytes;
1256   this(CharReader reader, char[] buffer, BufferTooSmall tooSmall)
1257   {
1258     this.reader = reader;
1259     this.chunker = LinesChunker(buffer, tooSmall);
1260     this.currentBytes = this.chunker.read(reader);
1261   }
1262   @property empty()
1263   {
1264     return currentBytes == 0;
1265   }
1266   @property char[] front() {
1267     return chunker.buffer[0..currentBytes];
1268   }
1269   @property void popFront() {
1270     this.currentBytes = this.chunker.read(reader);
1271   }
1272 }
1273 auto byLines(CharReader reader, char[] buffer, BufferTooSmall tooSmall) {
1274   return LinesReader(reader, buffer, tooSmall);
1275 }
1276 auto byLines(File file, char[] buffer, BufferTooSmall tooSmall) {
1277   auto fileCharReader = FileCharReader(file);
1278   return LinesReader(&(fileCharReader.read), buffer, tooSmall);
1279 }
1280 
1281 
1282 version(unittest_common) unittest
1283 {
1284   mixin(scopedTest!("LinesChunker/LinesReader"));
1285 
1286   CustomChunks customChunks;
1287   char[5] buffer5;
1288   char[256] buffer256;
1289 
1290   void testLinesChunkerCustom(string[] expectedChunks, CharReader reader, char[] chunkerBuffer = buffer256, size_t testLine = __LINE__)
1291   {
1292     auto lineChunker = LinesChunker(chunkerBuffer, BufferTooSmall.throwException);
1293 
1294     foreach(expectedChunk; expectedChunks) {
1295       size_t actualLength = lineChunker.read(reader);
1296       if(chunkerBuffer[0..actualLength] != expectedChunk) {
1297 	writefln("Expected: '%s'", escape(expectedChunk));
1298 	writefln("Actual  : '%s'", escape(chunkerBuffer[0..actualLength]));
1299 	assert(0);
1300       }
1301 /+
1302       debug {
1303 	writefln("
1304       }
1305 +/
1306     }
1307   }
1308   void testLinesChunker(string[] expectedChunks, string[] customChunkStrings, char[] chunkerBuffer = buffer256, size_t testLine = __LINE__)
1309   {
1310     writefln("testLinesChunker %s %s", expectedChunks, customChunkStrings);
1311 
1312     customChunks = CustomChunks(customChunkStrings);
1313     testLinesChunkerCustom(expectedChunks, &(customChunks.read), chunkerBuffer, testLine);
1314   }
1315   
1316   
1317   testLinesChunker(["a\n"], ["a\n"]);
1318   testLinesChunker(["a\n", "b"], ["a\nb"], buffer5); // NOTE: One would think that this should return one chunk, however,
1319                                                      //       This simulates the case when the reader returned the available
1320                                                      //       data so there's no way for the chunker to know if there is another newline
1321                                                      //       coming at the next read call so it just returns the currently known lines
1322 
1323   testLinesChunker(["a\nb\r\nc\n"], ["a\nb\r\nc\n"]);
1324   testLinesChunker(["a\nb\r\nc\n", "d"], ["a\nb\r\nc\nd"]);
1325   testLinesChunker(["123\n", "123\n"], ["123\n1", "23\n"], buffer5);
1326   testLinesChunker(["123\n", "123"], ["123\n1", "23"], buffer5);
1327 
1328   //
1329   // Test LinesReader
1330   //
1331 /+
1332   void testByLines(string[] expectedChunks, string[] customChunkStrings, char[] cunkBuffer = buffer256, size_t testLine = __LINE__)
1333   {
1334     
1335   }
1336 +/
1337   customChunks = CustomChunks(["a\nb"]);
1338   foreach(lines; (&(customChunks.read)).byLines(buffer5, BufferTooSmall.throwException)) {
1339     writefln("lines: '%s'", escape(lines));
1340   }
1341 }
1342 
1343 
1344 struct LineReader
1345 {
1346   CharReader reader;
1347   LinesChunker chunker;
1348 
1349   size_t dataLength;
1350   char[] line;
1351   size_t endOfLineOffset;
1352 
1353   this(CharReader reader, char[] buffer, BufferTooSmall tooSmall) {
1354     this.reader = reader;
1355     this.chunker = LinesChunker(buffer, tooSmall);
1356 
1357     this.dataLength = this.chunker.read(reader);
1358     this.line = null;
1359     this.endOfLineOffset = 0;
1360     popFront();
1361   }
1362   @property bool empty()
1363   {
1364     return line is null;
1365   }
1366   @property auto front()
1367   {
1368     return line;
1369   }
1370   @property void popFront()
1371   {
1372     if(endOfLineOffset >= dataLength) {
1373 
1374       if(this.line is null) {
1375 	//writefln("[DEBUG] LineReader.popFront no more data");
1376 	return;
1377       }
1378       this.dataLength = this.chunker.read(reader);
1379       if(this.dataLength == 0) {
1380 	this.line = null;
1381 	//writefln("[DEBUG] LineReader.popFront no more data");
1382 	return;
1383       }
1384       endOfLineOffset = 0;
1385     }
1386 
1387     auto startOfNextLine = endOfLineOffset;
1388     while(true) {
1389       if(chunker.buffer[endOfLineOffset] == '\n') {
1390 	endOfLineOffset++;
1391 	line = chunker.buffer[startOfNextLine..endOfLineOffset];
1392 	break;
1393       }
1394       endOfLineOffset++;
1395       if(endOfLineOffset >= dataLength) {
1396 	line = chunker.buffer[startOfNextLine..endOfLineOffset];
1397 	break;
1398       }
1399     }
1400 
1401     //writefln("[DEBUG] LineReader.popFront '%s'", escape(line));
1402 
1403   }
1404 }
1405 
1406 
1407 auto byLine(CharReader reader, char[] buffer, BufferTooSmall tooSmall) {
1408   return LineReader(reader, buffer, tooSmall);
1409 }
1410 /*
1411 auto byLine(File file, char[] buffer, BufferTooSmall tooSmall) {
1412   auto fileCharReader = FileCharReader(file);
1413   return LinesReader(&(fileCharReader.read), buffer, tooSmall);
1414 }
1415 */
1416 
1417 version(unittest_common) unittest
1418 {
1419   mixin(scopedTest!("LineReader"));
1420 
1421   char[256] buffer256;
1422 
1423   void testLines(string data, size_t testLine = __LINE__) {
1424     CustomChunks customChunks;
1425 	
1426     for(auto chunkSize = 1; chunkSize <= data.length; chunkSize++) {
1427 
1428       // Create Chunks
1429       string[] chunks;
1430       size_t offset;
1431       for(offset = 0; offset + chunkSize <= data.length; offset += chunkSize) {
1432 	chunks ~= data[offset .. offset + chunkSize];
1433       }
1434       if(data.length - offset > 0) {
1435 	chunks ~= data[offset .. $];
1436       }
1437       //writefln("ChunkSize %s Chunks %s", chunkSize, chunks);
1438 
1439       customChunks = CustomChunks(chunks);
1440       auto lineReader = LineReader(&(customChunks.read), buffer256, BufferTooSmall.throwException);
1441 
1442       //writefln("[DEBUG] lineReader.front = '%s'", lineReader.front);
1443       
1444       
1445       size_t lineNumber = 1;
1446       foreach(line; data.byLine()) {
1447 	//writefln("[DEBUG] line %s '%s'", lineNumber, escape(line));
1448 
1449 	if(lineReader.empty) {
1450 	  writefln("Expected line '%s' but no more lines", escape(line));
1451 	  assert(0);
1452 	}
1453 	if(line != lineReader.front) {
1454 	  writefln("Expected: '%s'", escape(line));
1455 	  writefln("Actual  : '%s'", escape(lineReader.front));
1456 	  assert(0);
1457 	}
1458 	lineReader.popFront;
1459 	lineNumber++;
1460       }
1461 
1462       if(!lineReader.empty) {
1463 	writefln("Got extra line '%s' but expected no more lines", escape(lineReader.front));
1464 	assert(0);
1465       }
1466 
1467     }
1468   }
1469 
1470   //testLines("abc");
1471   //testLines("abc\n");
1472   testLines("abc\n1234\n\n");
1473 
1474 
1475 }

Page generated by adrdox