all the slides for tomorrrow

2026-10-25 05:33:17 -06:00 · 2025-05-19 22:04:12 +02:00
1 parent 489beae2c5
commit 5984895de8
@@ -0,0 +1,6 @@
+1	the	the	DET	DT	Definite=Def|PronType=Art	3	det	_	TokenRange=0:3
+2	black	black	ADJ	JJ	Degree=Pos	3	amod	_	TokenRange=4:9
+3	cat	cat	NOUN	NN	Number=Sing	4	nsubj	_	TokenRange=10:13
+4	sees	see	VERB	VBZ	Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	0	root	_	TokenRange=14:18
+5	us	we	PRON	PRP	Case=Acc|Number=Plur|Person=1|PronType=Prs	4	obj	_	TokenRange=19:21
+6	now	now	ADV	RB	PronType=Dem	4	advmod	_	SpaceAfter=No|TokenRange=22:25
@@ -0,0 +1,51 @@
+<svg width="317"
+     height="115"
+     viewBox="0 0 317 115"
+     version="1.1"
+     xmlns="http://www.w3.org/2000/svg">
+  <text x="5" y="108" font-size="16">the</text>
+  <text x="42" y="108" font-size="16">black</text>
+  <text x="97" y="108" font-size="16">cat</text>
+  <text x="143" y="108" font-size="16">sees</text>
+  <text x="189" y="108" font-size="16">us</text>
+  <text x="235" y="108" font-size="16">now</text>
+  <text x="5" y="93" font-size="10">DET</text>
+  <text x="42" y="93" font-size="10">ADJ</text>
+  <text x="97" y="93" font-size="10">NOUN</text>
+  <text x="143" y="93" font-size="10">VERB</text>
+  <text x="189" y="93" font-size="10">PRON</text>
+  <text x="235" y="93" font-size="10">ADV</text>
+  <path d="M 17 80 Q 17 47 50 47 L 72 47 Q 105 47 105 80"
+        stroke="black"
+        fill="none"/>
+  <line x1="17" y1="75" x2="17" y2="80" stroke="black"/>
+  <path d="M 17 80 14 74 20 74"/>
+  <text x="54" y="42" font-size="10">det</text>
+  <path d="M 55 80 Q 55 63 71 63 L 88 63 Q 104 63 104 80"
+        stroke="black"
+        fill="none"/>
+  <line x1="55" y1="75" x2="55" y2="80" stroke="black"/>
+  <path d="M 55 80 52 74 58 74"/>
+  <text x="71" y="58" font-size="10">amod</text>
+  <path d="M 110 80 Q 110 63 127 63 L 133 63 Q 150 63 150 80"
+        stroke="black"
+        fill="none"/>
+  <line x1="110" y1="75" x2="110" y2="80" stroke="black"/>
+  <path d="M 110 80 107 74 113 74"/>
+  <text x="119" y="58" font-size="10">nsubj</text>
+  <line x1="158" y1="20" x2="158" y2="80" stroke="black"/>
+  <path d="M 158 80 155 74 161 74"/>
+  <text x="163" y="28" font-size="10">root</text>
+  <path d="M 166 80 Q 166 63 183 63 L 189 63 Q 206 63 206 80"
+        stroke="black"
+        fill="none"/>
+  <line x1="206" y1="75" x2="206" y2="80" stroke="black"/>
+  <path d="M 206 80 203 74 209 74"/>
+  <text x="179" y="58" font-size="10">obj</text>
+  <path d="M 165 80 Q 165 47 198 47 L 220 47 Q 253 47 253 80"
+        stroke="black"
+        fill="none"/>
+  <line x1="253" y1="75" x2="253" y2="80" stroke="black"/>
+  <path d="M 253 80 250 74 256 74"/>
+  <text x="195" y="42" font-size="10">advmod</text>
+</svg>
@@ -1,6 +1,6 @@
 ---
-title: "Training and evaluating UD parsers"
-subtitle: "by popular demand"
+title: "Training and evaluating \\newline dependency parsers"
+subtitle: "(added to the course by popular demand)"
 author: "Arianna Masciolini"
 theme: "lucid"
 logo: "gu.png"
@@ -8,7 +8,173 @@ date: "VT25"
 institute: "LT2214 Computational Syntax"
 ---

-# Basics of dependency parsing
+## Today's topic
+\bigskip \bigskip
+![](img/sets.png)

-## Today's focus
-<svg version="1.1" viewBox="0.0 0.0 960.0 720.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l960.0 0l0 720.0l-960.0 0l0 -720.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l960.0 0l0 720.0l-960.0 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m0 360.00787l0 0c0 -198.82251 202.92429 -360.0 453.24408 -360.0l0 0c250.31982 0 453.24408 161.17749 453.24408 360.0l0 0c0 198.82251 -202.92426 360.0 -453.24408 360.0l0 0c-250.3198 0 -453.24408 -161.17749 -453.24408 -360.0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m0 360.00787l0 0c0 -198.82251 202.92429 -360.0 453.24408 -360.0l0 0c250.31982 0 453.24408 161.17749 453.24408 360.0l0 0c0 198.82251 -202.92426 360.0 -453.24408 360.0l0 0c-250.3198 0 -453.24408 -161.17749 -453.24408 -360.0z" fill-rule="evenodd"/><path fill="#ffffff" d="m68.12775 330.84973l0 0c0 -150.0649 165.59824 -271.71655 369.87402 -271.71655l0 0c204.27576 0 369.87402 121.65164 369.87402 271.71655l0 0c0 150.06491 -165.59827 271.71655 -369.87402 271.71655l0 0c-204.27579 0 -369.87402 -121.65164 -369.87402 -271.71655z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m68.12775 330.84973l0 0c0 -150.0649 165.59824 -271.71655 369.87402 -271.71655l0 0c204.27576 0 369.87402 121.65164 369.87402 271.71655l0 0c0 150.06491 -165.59827 271.71655 -369.87402 271.71655l0 0c-204.27579 0 -369.87402 -121.65164 -369.87402 -271.71655z" fill-rule="evenodd"/><path fill="#ffffff" d="m120.255196 300.69324l0 0c0 -105.15155 114.39654 -190.3937 255.51181 -190.3937l0 0c141.11526 0 255.51181 85.24216 255.51181 190.3937l0 0c0 105.15152 -114.396545 190.3937 -255.51181 190.3937l0 0c-141.11526 0 -255.51181 -85.24219 -255.51181 -190.3937z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m120.255196 300.69324l0 0c0 -105.15155 114.39654 -190.3937 255.51181 -190.3937l0 0c141.11526 0 255.51181 85.24216 255.51181 190.3937l0 0c0 105.15152 -114.396545 190.3937 -255.51181 190.3937l0 0c-141.11526 0 -255.51181 -85.24219 -255.51181 -190.3937z" fill-rule="evenodd"/><path fill="#f2581a" d="m182.09082 269.72812l0 0c0 -56.011246 60.69185 -101.41733 135.55905 -101.41733l0 0c74.86719 0 135.55905 45.406082 135.55905 101.41733l0 0c0 56.01123 -60.691864 101.41733 -135.55905 101.41733l0 0c-74.8672 0 -135.55905 -45.406097 -135.55905 -101.41733z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m182.09082 269.72812l0 0c0 -56.011246 60.69185 -101.41733 135.55905 -101.41733l0 0c74.86719 0 135.55905 45.406082 135.55905 101.41733l0 0c0 56.01123 -60.691864 101.41733 -135.55905 101.41733l0 0c-74.8672 0 -135.55905 -45.406097 -135.55905 -101.41733z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m645.3995 548.7699l314.61414 0l0 78.86615l-314.61414 0z" fill-rule="evenodd"/><path fill="#000000" d="m661.6182 583.6099l0 10.0l-3.59375 0l0 -26.75l7.890625 0q2.53125 0 4.40625 0.59375q1.890625 0.59375 3.109375 1.671875q1.234375 1.078125 1.84375 2.609375q0.609375 1.53125 0.609375 3.421875q0 1.859375 -0.65625 3.40625q-0.65625 1.546875 -1.921875 2.671875q-1.25 1.125 -3.125 1.75q-1.859375 0.625 -4.265625 0.625l-4.296875 0zm0 -2.875l4.296875 0q1.546875 0 2.734375 -0.40625q1.1875 -0.421875 1.984375 -1.15625q0.8125 -0.734375 1.21875 -1.765625q0.40625 -1.03125 0.40625 -2.25q0 -2.5625 -1.578125 -4.0q-1.578125 -1.4375 -4.765625 -1.4375l-4.296875 0l0 11.015625zm31.264832 12.875l-1.46875 0q-0.484375 0 -0.796875 -0.140625q-0.296875 -0.15625 -0.390625 -0.640625l-0.359375 -1.75q-0.75 0.671875 -1.46875 1.203125q-0.703125 0.53125 -1.484375 0.890625q-0.78125 0.359375 -1.671875 0.546875q-0.890625 0.1875 -1.96875 0.1875q-1.109375 0 -2.078125 -0.296875q-0.953125 -0.3125 -1.65625 -0.921875q-0.703125 -0.625 -1.125 -1.5625q-0.421875 -0.953125 -0.421875 -2.234375q0 -1.125 0.609375 -2.15625q0.625 -1.03125 1.984375 -1.828125q1.375 -0.8125 3.59375 -1.328125q2.234375 -0.515625 5.4375 -0.59375l0 -1.46875q0 -2.203125 -0.953125 -3.328125q-0.9375 -1.140625 -2.78125 -1.140625q-1.203125 0 -2.046875 0.3125q-0.828125 0.3125 -1.4375 0.703125q-0.59375 0.375 -1.046875 0.6875q-0.4375 0.296875 -0.859375 0.296875q-0.34375 0 -0.59375 -0.171875q-0.25 -0.171875 -0.390625 -0.4375l-0.609375 -1.0625q1.578125 -1.515625 3.375 -2.265625q1.8125 -0.75 4.015625 -0.75q1.59375 0 2.828125 0.53125q1.234375 0.515625 2.0625 1.453125q0.84375 0.9375 1.265625 2.265625q0.4375 1.3125 0.4375 2.90625l0 12.09375zm-8.625 -2.03125q0.875 0 1.609375 -0.171875q0.734375 -0.1875 1.375 -0.515625q0.640625 -0.328125 1.21875 -0.78125q0.59375 -0.46875 1.15625 -1.078125l0 -3.9375q-2.296875 0.078125 -3.90625 0.375q-1.59375 0.28125 -2.609375 0.75q-1.0 0.46875 -1.46875 1.109375q-0.453125 0.6Line truncated
+# Parsing
+
+## A structured prediction task
+Sequence $\to$ structure, e.g.
+
+- natural language sentence $\to$ syntax tree
+- code $\to$ AST
+- argumentative essay $\to$ argumentative structure
+
+## Example (argmining)
+
+> Språkbanken has better fika than CLASP: every fika, someone bakes. Sure, CLASP has a better coffee machine. On the other hand, there are more important things than coffee. In fact, most people drink tea in the afternoon.
+
+## Example (argmining)
+![](img/argmining.png)
+
+\footnotesize From "A gentle introduction to argumentation mining" (Lindahl et al., 2022)
+
+# Syntactic parsing
+
+## From sentence to tree
+From Jurafsky & Martin. _Speech and Language Processing_, chapter 18 (January 2024 draft):
+
+> Syntactic parsing is the task of assigning a syntactic structure to a sentence
+
+- the structure is usually a _syntax tree_
+- two main classes of approaches:
+  - constituency parsing (e.g. GF)
+  - dependency parsing (e.g. UD)
+
+## Example (GF)
+```
+MicroLang> i MicroLangEng.gf 
+linking ... OK
+
+Languages: MicroLangEng
+7 msec
+MicroLang> p "the black cat sees us now"
+PredVPS (DetCN the_Det (AdjCN (PositA black_A) 
+(UseN cat_N))) (AdvVP (ComplV2 see_V2 (UsePron 
+we_Pron)) now_Adv)
+```
+
+## Example (GF)
+```haskell
+PredVPS (
+    DetCN 
+        the_Det 
+        (AdjCN (PositA black_A) (UseN cat_N))
+    ) 
+    (AdvVP 
+        (ComplV2 see_V2 (UsePron we_Pron)) 
+        now_Adv
+    )
+```
+
+## Example (GF) 
+![](img/gfast.png)
+
+# Dependency parsing
+
+## Example (UD)
+![](img/ud.svg)
+
+\small
+```
+1	the	_	DET	_	_	3	det	_	_
+2	black	_	ADJ	_	_	3	amod	_	_
+3	cat	_	NOUN	_	_	4	nsubj	_	_
+4	sees	_	VERB	_	_	0	root	_	_
+5	us	_	PRON	_	_	4	obj	_	_
+6	now	_	ADV	_	_	4	advmod	_	_
+```
+
+## Two paradigms
+- __graph-based algorithms__: find the optimal tree from the set of all possible candidate solutions or a subset of it
+- __transition-based algorithms__: incrementally build a tree by solving a sequence of classification problems
+
+## Graph-based approaches
+$$\hat{t} = \underset{t \in T(s)}{argmax}\, score(s,t)$$
+
+- $t$: candidate tree
+- $\hat{t}$: predicted tree
+- $s$: input sentence
+- $T(s)$: set of candidate trees for $s$ 
+
+## Complexity
+- choice of $T$ (upper bound: $n^{n-1}$, where $n$ is the number of words in $s$)
+- scoring function (in the __arc-factor model__, the score of a tree is the sum of the score of each edge, scored individually by a NN. This results in $O(n^3)$ complexity)
+
+## Transition-based approaches
+- trees are built through a sequence of steps, called _transitions_
+- training requires:
+  - a gold-standard treebank (as for graph-based approaches)
+  - an _oracle_ i.e. an algorithm that converts each tree into a a gold-standard sequence of transitions
+- much more efficient: $O(n)$
+
+## Evaluation
+2 main metrics:
+
+- __UAS__ (Unlabelled Attachment Score): what's the fraction of nodes are attached to the correct dependency head?
+- __LAS__ (Labelled Attachment Score): what's the fraction of nodes are attached to the correct dependency head _with an arc labelled with the correct relation type_[^1]?
+
+[^1]: in UD: the `DEPREL` column
+
+# Specifics of UD parsing
+
+## Not just parsing per se
+UD "parsers" typically do a lot more than just dependency parsing:
+
+- lemmatization (`LEMMA` column)
+- POS tagging (`UPOS` + `XPOS`)
+- morphological tagging (`FEATS`)
+- ...
+
+## Evaluation (UD-specific)
+Some more specific metrics:
+
+- CLAS (Content-word LAS): LAS limited to content words
+- MLAS (Morphology-Aware LAS): CLAS that also uses the `FEATS` column
+- BLEX (Bi-Lexical dependency score): CLAS that also uses the `LEMMA` column
+
+## Evaluation script output
+\small
+```
+Metric     | Precision |    Recall |  F1 Score | AligndAcc
+-----------+-----------+-----------+-----------+-----------
+Tokens     |    100.00 |    100.00 |    100.00 |
+Sentences  |    100.00 |    100.00 |    100.00 |
+Words      |    100.00 |    100.00 |    100.00 |
+UPOS       |     98.36 |     98.36 |     98.36 |     98.36
+XPOS       |    100.00 |    100.00 |    100.00 |    100.00
+UFeats     |    100.00 |    100.00 |    100.00 |    100.00
+AllTags    |     98.36 |     98.36 |     98.36 |     98.36
+Lemmas     |    100.00 |    100.00 |    100.00 |    100.00
+UAS        |     92.73 |     92.73 |     92.73 |     92.73
+LAS        |     90.30 |     90.30 |     90.30 |     90.30
+CLAS       |     88.50 |     88.34 |     88.42 |     88.34
+MLAS       |     86.72 |     86.56 |     86.64 |     86.56
+BLEX       |     88.50 |     88.34 |     88.42 |     88.34
+```
+
+## Three generations of parsers
+1. __MaltParser__ (Nivre et al., 2006): "classic" transition-based parser, data-driven but not NN-based
+2. __UDPipe__: neural transition-based parser; personal favorite
+   - version 1 (Straka et al. 2016): solid and fast software, available anywhere
+   - version 2 (Straka et al. 2018): much better performance, but slower and only available through an API
+3. __MaChAmp__ (van der Goot et al., 2021): transformer-based toolkit for multi-task learning, works on all CoNNL-like data, close to the SOTA, relatively easy to install and train
+
+## Your task (lab 3)
+![](img/machamp.png)
+
+1. annotate a small treebank for your language of choice (started)
+2. train a parser-tagger with MaChAmp on a reference UD treebank (tomorrow: installation)
+3. evaluate it on your treebank
+
+## Sources/further reading
+- chapters 18-19 of the January 2024 draft of _Speech and Language Processing_ (Jurafsky & Martin) (full text available [__here__](https://web.stanford.edu/~jurafsky/slp3/))
+- unit 3-2 of Johansson & Kuhlmann's course "Deep Learning for Natural Language Processing" (slides and videos available __[__here__](https://liu-nlp.ai/dl4nlp/modules/module3/)__)
+- section 10.9.2 on parser evaluation from Aarne's course notes (on Canvas or [__here__](https://www.cse.chalmers.se/~aarne/grammarbook.pdf))
+
+## Papers describing the parsers
+- _MaltParser: A Data-Driven Parser-Generator for Dependency Parsing_ (Nivre et al. 2006) (PDF [__here__](http://lrec-conf.org/proceedings/lrec2006/pdf/162_pdf.pdf))
+- _UDPipe: Trainable Pipeline for Processing CoNLL-U Files Performing Tokenization, Morphological Analysis, POS Tagging and Parsing_ (Straka et al. 2016) (PDF [__here__](https://aclanthology.org/L16-1680.pdf))
+- _UDPipe 2.0 Prototype at CoNLL 2018 UD Shared Task_ (Straka et al. 2018) (PDF [__here__](https://aclanthology.org/K18-2020.pdf))
+- _Massive Choice, Ample Tasks (MACHAMP): A Toolkit for Multi-task Learning in NLP_ (van der Goot et al., 2021) (PDF [__here__](https://arxiv.org/pdf/2005.14672))