From 87dc29dda3f828a808dd27df8575a65da96d5dc0 Mon Sep 17 00:00:00 2001
From: Oliver Marks <oly@digitaloctave.com>
Date: Sat, 3 Nov 2018 14:18:29 +0000
Subject: [PATCH] Add missing tokens file

---
 eorg/tokens.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 eorg/tokens.py

diff --git a/eorg/tokens.py b/eorg/tokens.py
new file mode 100644
index 0000000..0af1879
--- /dev/null
+++ b/eorg/tokens.py
@@ -0,0 +1,30 @@
+BLANK = 0
+META = 1
+HEADER = 2
+BOLD = 10
+ITALIC = 11
+UNDERLINED = 12
+VERBATIM = 13
+LIST = 20
+TEXT = 21
+IMAGE = 22
+LINK = 23
+CAPTION = 24
+SOURCE = 50
+EXAMPLE = 51
+RESULTS = 52
+COMMENT = 53
+
+
+
+class Token:
+    __slots__ = ["token", "value", "attrs"]
+
+    def __init__(self, token, value="", attrs=None):
+        self.token = token
+        self.value = value
+        self.attrs = attrs
+
+    def __repr__(self):
+        return f'Token(token="{self.token}", value="{self.value}", attrs="{self.attrs}")'
+