Merge commit 'ad273dbe5dba7bd0e901270464e25fc1f030a5b5' as 'backend/goldmark'

backend/goldmark/_tools/gen-emb-structs.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func embStructsSubCommand(args []string) {
+	cmdName := "emb-structs"
+	cmd := flag.NewFlagSet(cmdName, flag.ExitOnError)
+	cmd.Usage = func() {
+		_, _ = fmt.Fprintf(cmd.Output(), `Usage of %s:
+
+  Embed Go structs as compact format in the binary.
+  JSON file format:
+    {
+      "prefix": "html5entities", // prefix for the generated const/var
+      "types": { // struct field types
+          "Name": "string",         // struct fields
+          "CodePoints": "[]int",
+          "Characters": "[]byte"
+      },
+
+      // array of struct data
+      // all data must be a string
+      "data": [ 
+        {"Name": "AElig", "CodePoints": ["198"], "Characters": ["0xc3", "0x86"]},
+        {"Name": "AMP", "CodePoints": ["38"], "Characters": ["0x26"]},
+      ]
+    }
+
+`, cmdName)
+		cmd.PrintDefaults()
+	}
+
+	inputJSONPath := cmd.String("i", "", "source JSON file path(required)")
+	outputPath := cmd.String("o", "", "output file path(required)")
+
+	if err := cmd.Parse(args); err != nil ||
+		len(*inputJSONPath) == 0 ||
+		len(*outputPath) == 0 {
+		usage(cmd.Usage, err)
+	}
+	var source map[string]any
+	inputJSONSource, err := os.ReadFile(*inputJSONPath)
+	if err != nil {
+		panic(err)
+	}
+	if err := json.Unmarshal(inputJSONSource, &source); err != nil {
+		panic(err)
+	}
+
+	f, err := os.Create(*outputPath)
+	if err != nil {
+		fmt.Printf("Failed to open %s: %v\n", *outputPath, err)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	abs, _ := filepath.Abs(*outputPath)
+	pkg := filepath.Base(filepath.Dir(abs))
+
+	write := func(ft string, v ...any) {
+		if len(v) == 0 {
+			_, _ = f.WriteString(ft)
+			return
+		}
+		_, _ = f.WriteString(fmt.Sprintf(ft, v...))
+	}
+	writeln := func(ft string, v ...any) {
+		write(ft+"\n", v...)
+	}
+
+	writeln("// Code generated by _tools; DO NOT EDIT.")
+	writeln("package " + pkg)
+
+	prefix := source["prefix"].(string)
+	types := source["types"].(map[string]any)
+	data := source["data"].([]any)
+
+	writeln("const _%sLength = %d", prefix, len(data))
+
+	for prop, _typ := range types {
+		typ := _typ.(string)
+		if typ == "string" {
+			write("const _%s%s string = \"", prefix, prop)
+			for _, _d := range data {
+				d := _d.(map[string]any)
+				v := d[prop].(string)
+				write(v)
+			}
+			writeln(`"`)
+			write("const _%s%sIndex  = \"", prefix, prop)
+			for _, _d := range data {
+				d := _d.(map[string]any)
+				v := d[prop].(string)
+				write("\\x%02x", len(v))
+			}
+			writeln(`"`)
+			continue
+		}
+
+		if strings.HasPrefix(typ, "[]") {
+			elmTyp := typ[2:]
+			write("var _%s%s = [...]%s{", prefix, prop, elmTyp)
+			for i, _d := range data {
+				d := _d.(map[string]any)
+				arr := d[prop].([]any)
+				for j, a := range arr {
+					v := a.(string)
+					write(v)
+					if i != len(data)-1 || j != len(arr)-1 {
+						write(", ")
+					}
+				}
+			}
+			writeln("}")
+			write("var _%s%sIndex  = \"", prefix, prop)
+			for _, _d := range data {
+				d := _d.(map[string]any)
+				arr := d[prop].([]any)
+				write("\\x%02x", len(arr))
+			}
+			writeln(`"`)
+			continue
+		}
+
+		write("var _%s%s = [...]%s{", prefix, prop, typ)
+		for i, _d := range data {
+			d := _d.(map[string]any)
+			v := d[prop].(string)
+			write(v)
+			if i != len(data)-1 {
+				write(", ")
+			}
+		}
+		writeln(`}`)
+	}
+}

backend/goldmark/_tools/gen-oss-fuzz-corpus.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type TestCase struct {
+	Example  int    `json:"example"`
+	Markdown string `json:"markdown"`
+}
+
+func ossFuzzCorpusSubCommand(args []string) {
+	corpus_out := args[0]
+	if !strings.HasSuffix(corpus_out, ".zip") {
+		log.Fatalln("Expected command line:", os.Args[0], "<corpus_output>.zip")
+	}
+
+	zip_file, err := os.Create(corpus_out)
+
+	zip_writer := zip.NewWriter(zip_file)
+
+	if err != nil {
+		log.Fatalln("Failed creating file:", err)
+	}
+
+	json_corpus := "_test/spec.json"
+	bs, err := ioutil.ReadFile(json_corpus)
+	if err != nil {
+		log.Fatalln("Could not open file:", json_corpus)
+		panic(err)
+	}
+	var testCases []TestCase
+	if err := json.Unmarshal(bs, &testCases); err != nil {
+		panic(err)
+	}
+
+	for _, c := range testCases {
+		file_in_zip := "example-" + strconv.Itoa(c.Example)
+		f, err := zip_writer.Create(file_in_zip)
+		if err != nil {
+			log.Fatal(err)
+		}
+		_, err = f.Write([]byte(c.Markdown))
+		if err != nil {
+			log.Fatalf("Failed to write file: %s into zip file", file_in_zip)
+		}
+	}
+
+	err = zip_writer.Close()
+	if err != nil {
+		log.Fatal("Failed to close zip writer", err)
+	}
+
+	zip_file.Close()
+}

backend/goldmark/_tools/gen-unicode-case-folding-map.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type caseFolding struct {
+	Class byte
+	From  rune
+	To    []rune
+}
+
+func unicodeCaseFoldingMapSubCommand(args []string) {
+	cmdName := "unicode-case-folding-map"
+	cmd := flag.NewFlagSet(cmdName, flag.ExitOnError)
+	cmd.Usage = func() {
+		_, _ = fmt.Fprintf(cmd.Output(), `Usage of %s:
+
+  Generate input JSON data for emb-structs subcommand from unicode.org
+
+`, cmdName)
+		cmd.PrintDefaults()
+	}
+
+	outputPath := cmd.String("o", "", "output file path(required)")
+	unicodeVersion := cmd.String("u", "15.0.0", "unicode version")
+
+	if err := cmd.Parse(args); err != nil ||
+		len(*outputPath) == 0 {
+		usage(cmd.Usage, err)
+	}
+
+	url := "http://www.unicode.org/Public/" + *unicodeVersion + "/ucd/CaseFolding.txt"
+
+	resp, err := http.Get(url)
+	if err != nil {
+		fmt.Printf("Failed to get CaseFolding.txt: %v\n", err)
+		os.Exit(1)
+	}
+	defer resp.Body.Close()
+
+	bs, err := io.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Printf("Failed to get CaseFolding.txt: %v\n", err)
+		os.Exit(1)
+	}
+
+	buf := bytes.NewBuffer(bs)
+	scanner := bufio.NewScanner(buf)
+
+	embstructmap := make(map[string]any)
+	embstructmap["prefix"] = "unicodeCaseFolding"
+	embstructmap["types"] = map[string]any{
+		"From": "rune",
+		"To":   "[]rune",
+	}
+	var data []any
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") || len(strings.TrimSpace(line)) == 0 {
+			continue
+		}
+		line = strings.Split(line, "#")[0]
+		parts := strings.Split(line, ";")
+		for i, p := range parts {
+			parts[i] = strings.TrimSpace(p)
+		}
+		cf := caseFolding{}
+		v, _ := strconv.ParseInt(parts[0], 16, 32)
+		cf.From = rune(int32(v))
+		cf.Class = parts[1][0]
+		for _, v := range strings.Split(parts[2], " ") {
+			c, _ := strconv.ParseInt(v, 16, 32)
+			cf.To = append(cf.To, rune(int32(c)))
+		}
+		if cf.Class != 'C' && cf.Class != 'F' {
+			continue
+		}
+		var tos []string
+		for _, v := range cf.To {
+			tos = append(tos, fmt.Sprintf("%d", v))
+		}
+		data = append(data, map[string]any{
+			"From": fmt.Sprintf("0x%x", cf.From),
+			"To":   tos,
+		})
+	}
+	embstructmap["data"] = data
+	jsonData, err := json.MarshalIndent(embstructmap, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	err = os.WriteFile(*outputPath, jsonData, 0644)
+	if err != nil {
+		panic(err)
+	}
+
+}

backend/goldmark/_tools/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+
+	var args []string
+	cmd := "-h"
+	if len(os.Args) > 1 {
+		cmd = os.Args[1]
+	}
+	if len(os.Args) > 2 {
+		args = os.Args[2:]
+	}
+
+	switch cmd {
+	case "oss-fuzz-corpus":
+		ossFuzzCorpusSubCommand(args)
+	case "unicode-case-folding-map":
+		unicodeCaseFoldingMapSubCommand(args)
+	case "emb-structs":
+		embStructsSubCommand(args)
+	case "-h":
+		fallthrough
+	default:
+		fmt.Fprintf(os.Stderr, `Usage: _tools <subcommand> [options]
+subcommands:
+  oss-fuzz-corpus
+  unicode-case-folding-map
+  emb-structs
+`)
+		os.Exit(1)
+	}
+}
+
+func usage(u func(), err error) {
+	u()
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+	}
+	os.Exit(1)
+}