$ brew install curl python python3
$ pip install virtualenv

$ sudo apt install curl python2.7-dev python3-dev
$ sudo -H pip install virtualenv

$ virtualenv --python=python3 ve
$ . ve/bin/activate
$ pip3 install nltk numpy scipy scikit-learn gensim spaCy
$ python3 -m spacy.en.download
$ python3
>>> import nltk
>>> nltk.download_shell()

$ curl http://www.gutenberg.org/cache/epub/100/pg100.txt \
  | gzcat \
  > /tmp/shakes.txt

$ wc /tmp/shakes.txt
  124787  904061 5589889 /tmp/shakes.txt

$ cat /tmp/shakes.txt \
| sed $'s/./&\\\n/g' \
| awk '{cnt[$0] += 1} END {for (i in cnt) print cnt[i], i}' \
| sort -nr

#!/usr/bin/env python

import argparse
import codecs
import collections
import re
import sys
from operator import itemgetter

ENCODING = 'utf-8'

sys.stdin = codecs.getreader(ENCODING)(sys.stdin)
sys.stdout = codecs.getwriter(ENCODING)(sys.stdout)
sys.stderr = codecs.getwriter(ENCODING)(sys.stderr)

parser = argparse.ArgumentParser()
parser.add_argument('-n',
                    dest='n',
                    type=int,
                    required=True)
args = parser.parse_args()
count = collections.defaultdict(int)

clean_input = re.compile(r'[\t\n]').sub(' ', sys.stdin.read())

for idx in range(0, len(clean_input) - args.n):
    count[clean_input[idx:idx + args.n]] += 1

for ngram, cnt in reversed(sorted(count.iteritems(), key=itemgetter(1))):
    print(u'{}\t{}'.format(cnt, ''.join(ngram)))

$ python3
>>> import nltk
>>> from nltk.metrics.distance import edit_distance
>>> edit_distance("foo", "food")
1
>>> edit_distance("foo", "fop")
1
>>> edit_distance("foo", "ofo")
2
>>> edit_distance("foo", "ofo", True)
1
>>> edit_distance("foo", "oof", True)
2

$ python3
>>> import re
>>> RX_WORD = re.compile(r'^\w+$')
>>> RX_WORD.search('λαμβδα')
<_sre.SRE_Match object; span=(0, 6), match='λαμβδα'>
>>> RX_ASCII_WORD = re.compile(r'^\w+$', re.A)
>>> RX_ASCII_WORD.search('λαμβδα')

$ python3
>>> import nltk
>>> nltk.tokenize.word_tokenize("Don't eat the yellow snow.")
['Do', "n't", 'eat', 'the', 'yellow', 'snow', '.']

$ python3
>>> import spacy
>>> nlp = spacy.en.English()
>>> doc = nlp("Don't eat the yellow snow.")
>>> print([str(tok) for tok in doc])
['Do', "n't ", 'eat ', 'the ', 'yellow ', 'snow', '.']

    0-9
    A-Za-z
    !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~

>>> for ch in "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~":
...   nltk.tokenize.word_tokenize(
...      ' ' + ch + 'foo' + ch + 'bar' + ch + ' ')
...
['!', 'foo', '!', 'bar', '!']
['``', 'foo', "''", 'bar', "''"]
['#', 'foo', '#', 'bar', '#']
['$', 'foo', '$', 'bar', '$']
['%', 'foo', '%', 'bar', '%']
['&', 'foo', '&', 'bar', '&']
["'foo'bar", "'"]
['(', 'foo', '(', 'bar', '(']
[')', 'foo', ')', 'bar', ')']
['*foo*bar*']
['+foo+bar+']
[',', 'foo', ',', 'bar', ',']
['-foo-bar-']
['.foo.bar', '.']
['/foo/bar/']
[':', 'foo', ':', 'bar', ':']
[';', 'foo', ';', 'bar', ';']
['<', 'foo', '<', 'bar', '<']
['=foo=bar=']
['>', 'foo', '>', 'bar', '>']
['?', 'foo', '?', 'bar', '?']
['@', 'foo', '@', 'bar', '@']
['[', 'foo', '[', 'bar', '[']
['\\foo\\bar\\']
[']', 'foo', ']', 'bar', ']']
['^foo^bar^']
['_foo_bar_']
['`foo`bar`']
['{', 'foo', '{', 'bar', '{']
['|foo|bar|']
['}', 'foo', '}', 'bar', '}']
['~foo~bar~']

*+-/=\^_`|~

!#$%&(),:;<>?@[]{}

"'.

tr -C '0-9a-zA-Z\047\052\053\055\057\075\134\136\137\140\174\176' ' '

$ tr -c '0-9a-zA-Z\047\052\053\055\057\075\134\136\137\140\174\176' ' ' \
  < /tmp/shakes.txt \
  | tr -s ' ' | tr ' ' '\n' | sort | uniq -c | sort -nr

$ python3 -c 'import nltk; print(" ".join(nltk.corpus.brown.words()))' \
  > /tmp/brown.txt

$ tr -c '0-9a-zA-Z\047\052\053\055\057\075\134\136\137\140\174\176' ' ' \
  < /tmp/brown.txt \
  | tr -s ' ' | tr ' ' '\n' | sort | uniq -c | sort -nr

>>> import nltk
>>> doc = "Sally sells sea shells by the sea shore."
>>> toks = nltk.tokenize.word_tokenize(doc)
>>> toks
['Sally', 'sells', 'sea', 'shells', 'by', 'the', 'sea', 'shore', '.']
>>> toks.count("sea")
2

>>> import collections
>>> bow = collections.defaultdict(int)
>>> for toks in toks:
...     bow[tok] += 1
...
>>> bow
defaultdict(<type 'int'>, {'shells': 1, 'by': 1, 'shore': 1,
   'sea': 2, 'the': 1, '.': 1, 'Sally': 1, 'sells': 1})
>>> bow["sea"]
2

>>> import gensim
>>> docs = ["Sally sells sea shells by the sea shore.",
...   "It was many and many a year ago,\nIn a kingdom by the sea,"]
>>> docs
['Sally sells sea shells by the sea shore.',
   'It was many and many a year ago,\nIn a kingdom by the sea,']
>>> bows = [nltk.tokenize.word_tokenize(doc) for doc in docs]
>>> dictionary = gensim.corpora.Dictionary(bows)
>>> corpus = [dictionary.doc2bow(bow) for bow in bows]
>>> corpus
[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 2), (5, 1), (6, 1), (7, 1)],
   [(4, 1), (5, 1), (7, 1), (8, 1), (9, 2), (10, 1), (11, 2),
    (12, 1), (13, 1), (14, 2), (15, 1), (16, 1), (17, 1)]]

>>> dictionary.id2token
{0: u'shells', 1: u'sells', 2: u'.', 3: u'shore', 4: u'sea', 5: u'the',
 6: u'Sally', 7: u'by', 8: u'and', 9: u'a', 10: u'ago', 11: u'many',
 12: u'year', 13: u'It', 14: u',', 15: u'In', 16: u'kingdom',
 17: u'was'}
>>> dictionary.id2token[4]
u'sea'

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> docs = ["Sally sells sea shells by the sea shore.",
...   "It was many and many a year ago,\nIn a kingdom by the sea,"]
>>> vectorizer = CountVectorizer()
>>> counts = vectorizer.fit_transform(docs)
>>> print(counts)
  (0, 7)	1
  (0, 9)	1
  (0, 8)	2
  (0, 10)	1
  (0, 2)	1
  (0, 12)	1
  (0, 11)	1
  (1, 8)	1
  (1, 2)	1
  (1, 12)	1
  (1, 4)	1
  (1, 13)	1
  (1, 6)	2
  (1, 1)	1
  (1, 14)	1
  (1, 0)	1
  (1, 3)	1
  (1, 5)	1
>>> vectorizer.vocabulary_
{u'and': 1, u'ago': 0, u'shells': 10, u'many': 6, u'year': 14,
   u'sally': 7, u'it': 4, u'sells': 9, u'shore': 11, u'sea': 8,
   u'in': 3, u'kingdom': 5, u'the': 12, u'was': 13, u'by': 2}

$ echo 'foo bar is a bad worke' \
| tr ' ' '\n' \
| sort \
| comm -23 - /usr/share/dict/words
worke

$ brew install aspell
$ echo 'foo bar is a bad worke' | aspell list
worke

$ curl http://clarkgrubb.wdfiles.com/local--files/nlp/getty.txt \
> /tmp/getty.txt

$ python3
>>> import nltk
>>> from nltk.tokenize import sent_tokenize
>>> with open('/tmp/getty.txt') as f:
...   s = f.read()
...
>>> len(sent_tokenize(s))
10

$ python3
>>> import spacy
>>> nlp = spacy.en.English()
>>> doc = nlp(open('/tmp/getty.txt').read())
>>> len(list(doc.sents))
11

$ python3
>>> import nltk
>>> from nltk.stem.porter import PorterStemmer
>>> stemmer = PorterStemmer()
>>> stemmer.stem('walking')
'walk'

$ python3
>>> import nltk
>>> len(nltk.corpus.stopwords.words('english'))
127

$ python3 -c 'import nltk; '\
'print("\n".join(nltk.corpus.stopwords.words("english")))'

$ curl http://clarkgrubb.wdfiles.com/local--files/nlp/getty.txt \
> /tmp/getty.txt

$ python3
>>> from nltk.util import ngrams
>>> import nltk
>>> f = open('/tmp/getty.txt')
>>> tokens = nltk.tokenize.word_tokenize(f.read())
>>> fout = open('/tmp/ngrams.txt', 'w')
>>> bigrams = ngrams(tokens, 2)
>>> for p in bigrams:
...  fout.write('|'.join(p) + '\n')
...
>>> fout.close()

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> getty = open('/tmp/getty.txt').read()
>>> vectorizer = CountVectorizer(ngram_range=(2,2))
>>> analyzer = vectorizer.build_analyzer()
>>> bigrams = analyzer(getty)
>>> bigrams[0:10]
['fourscore and', 'and seven', 'seven years', 'years ago',
 'ago our', 'our fathers', 'fathers brought', 'brought forth',
 'forth on', 'on this']

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> getty = open('/tmp/getty.txt').read()
>>> vectorizer = CountVectorizer(ngram_range=(2,2))
>>> X = vectorizer.fit_transform([getty])

>>> pairs = list(zip(vectorizer.get_feature_names(), X.toarray()[0]))
>>> pairs[0:10]
[('above our', 1), ('add or', 1), ('advanced it', 1), ('ago our', 1),
 ('all men', 1), ('altogether fitting', 1), ('and dead', 1),
 ('and dedicated', 1), ('and proper', 1), ('and seven', 1)]

$ python3
>>> from sklearn.feature_extraction.text import TfidfVectorizer
>>> docs = [
...   'In Xanadu did Kubla Khan',
...   'A stately pleasure-dome decree:',
...   'Where Alph, the sacred river, ran',
...   'Through caverns measureless to man',
...   'Down to a sunless sea.',
...   'So twice five miles of fertile ground',
...   'With walls and towers were girdled round;',
...   'And there were gardens bright with sinuous rills,',
...   'Where blossomed many an incense-bearing tree;',
...   'And here were forests ancient as the hills,',
...   'Enfolding sunny spots of greenery.']
...
>>> vectorizer = TfidfVectorizer()
>>> data = vectorizer.fit_transform(docs)
>>> data.toarray()

$ python3
>>> import nltk
>>> from nltk.stem.wordnet import WordNetLemmatizer
>>> wnl = WordNetLemmatizer()
>>> wnl.lemmatize('dogs')
'dog'

>>> from nltk.corpus import wordnet
>>> wordnet.synsets('dog')
[Synset('dog.n.01'), Synset('frump.n.01'), Synset('dog.n.03'),
 Synset('cad.n.01'), Synset('frank.n.02'), Synset('pawl.n.01'),
 Synset('andiron.n.01'), Synset('chase.v.01')]

>>> dog = wordnet.synset('dog.n.01')
>>> dog.definition()
'''a member of the genus Canis (probably descended from the
 common wolf) that has been domesticated by man since
 prehistoric times; occurs in many breeds'''
>>> dog.lemmas()
[Lemma('dog.n.01.dog'), Lemma('dog.n.01.domestic_dog'),
 Lemma('dog.n.01.Canis_familiaris')]

>>> dog = wordnet.synset('dog.n.01')
>>> dog.hypernyms()
[Synset('canine.n.02'), Synset('domestic_animal.n.01')]
>>> len(dog.hyponyms())
18

>>> car = wordnet.synset('car.n.01')
>>> car.part_meronyms()
[Synset('accelerator.n.01'), Synset('air_bag.n.01'),
Synset('auto_accessory.n.01'), Synset('automobile_engine.n.01'),
Synset('automobile_horn.n.01'), Synset('buffer.n.06'),
Synset('bumper.n.02'), Synset('car_door.n.01'),
Synset('car_mirror.n.01'), Synset('car_seat.n.01'),
Synset('car_window.n.01'), Synset('fender.n.01'),
Synset('first_gear.n.01'), Synset('floorboard.n.02'),
Synset('gasoline_engine.n.01'), Synset('glove_compartment.n.01'),
Synset('grille.n.02'), Synset('high_gear.n.01'), Synset('hood.n.09'),
Synset('luggage_compartment.n.01'), Synset('rear_window.n.01'),
Synset('reverse.n.02'), Synset('roof.n.02'),
Synset('running_board.n.01'), Synset('stabilizer_bar.n.01'),
Synset('sunroof.n.01'), Synset('tail_fin.n.02'),
Synset('third_gear.n.01'), Synset('window.n.02')]

$ python3
>>> import nltk
>>> s = "Don't eat the yellow snow."
>>> nltk.pos_tag(nltk.tokenize.word_tokenize(s))
[('Do', 'NNP'), ("n't", 'RB'), ('eat', 'VB'), ('the', 'DT'),
 ('yellow', 'JJ'), ('snow', 'NN'), ('.', '.')]

$ python3
>>> import spacy
>>> nlp = spacy.en.English()
>>> doc = nlp("Don't eat the yellow snow.")
>>> [nlp.vocab.strings[doc[i].tag] for i in range(0, len(list(doc)))]
['VB', 'RB', 'VB', 'DT', 'JJ', 'NN', '.']

$ curl http://nlp.stanford.edu/software/stanford-parser-full-2015-01-29.zip > stanford-parser-full-2015-01-29.zip
$ unzip stanford-parser-full-2015-01-29.zip
$ cd stanford-parse-full-2015-01-30
$ echo $'How many birds are sitting in the tree?' > /tmp/test.txt
$ ./lexparser.sh /tmp/test.txt 2> /dev/null
(ROOT
  (SBARQ
    (WHNP
      (WHADJP (WRB How) (JJ many))
      (NNS birds))
    (SQ
      (VP (VBP are)
        (VP (VBG sitting)
          (PP (IN in)
            (NP (DT the) (NN tree))))))
    (. ?)))

advmod(many-2, How-1)
amod(birds-3, many-2)
nsubj(sitting-5, birds-3)
aux(sitting-5, are-4)
root(ROOT-0, sitting-5)
det(tree-8, the-7)
prep_in(sitting-5, tree-8)

(ROOT
  (S
    (VP (VB Make)
      (NP (PRP it))
      (ADVP (RB so)))
    (. !)))

root(ROOT-0, Make-1)
dobj(Make-1, it-2)
advmod(Make-1, so-3)

$ cat tree.py
def get_root(doc):
    tok = doc[0]
    while tok != tok.head:
        tok = tok.head

    return tok


def print_node(nlp, node, indent):
    print((' ' * indent) + nlp.vocab.strings[node.tag] + ' ' + str(node))
    for tok in node.children:
        if tok != node:
            print_node(nlp, tok, indent + 2)


def print_doc(nlp, doc):
    root = get_root(doc)
    print_node(nlp, root, indent=0)

$ python3
>>> import spacy
>>> nlp = spacy.en.English()
>>> doc = nlp('How many birds are sitting in the tree?\nMake it so!')
>>> import tree
>>> tree.print_doc(nlp, doc)
VBG sitting
  NNS birds
    JJ many
      WRB How
  VBP are
  IN in
    NN tree
      DT the
  . ?

>>> import nltk
>>> def features(s):
...     vec = {}
...     for token in nltk.tokenize.word_tokenize(s):
...         vec[token] = 1 if token not in vec else vec[token] + 1
...
...     return vec
...
>>> data = [('Hi John, I will see you at 9:00pm', 'ham'),
...         ('Buy Viagra at low, low prices.  One time offer.', 'spam'),
...         ('Your Amazon Order is for the book "Harry Potter"', 'ham'),
...         ('Earn a million dollars by working at home.', 'spam')]
...
>>> processed_data = [(features(tup[0]), tup[1]) for tup in data]
>>> classifier = nltk.NaiveBayesClassifier.train(processed_data)
>>> classifier.classify(features("This is good Viagra!"))
'spam'
>>> classifier.classify(features("We will meet you at 7pm.  Take care"))
'ham'

$ python3
>>> from nltk.corpus import reuters
>>> CATEGORIES = ['coffee', 'housing', 'money-supply', 'rubber']
>>> data = []
>>> for cat in CATEGORIES:
...    data.extend([{'label': cat, 'document': reuters.raw(fid)}
...       for fid
...       in reuters.fileids(cat)])
...
>>> len(data)
382
>>> import collections
>>> cat_to_cnt = collections.defaultdict(int)
>>> for row in data:
...   cat_to_cnt[row['label']] += 1
...
>>> cat_to_cnt
defaultdict(<class 'int'>, {'housing': 20, 'coffee': 139,
    'money-supply': 174, 'rubber': 49})

>>> docs = [d['document'] for d in data]
>>> from sklearn.feature_extraction.text import TfidfVectorizer
>>> vectorizer = TfidfVectorizer()
>>> tfidfs = vectorizer.fit_transform(docs)

>>> import numpy
>>> tfidfs.toarray().shape
(382, 5492)
>>> u, s, v = numpy.linalg.svd(tfidfs.toarray())

>>> sum(s[:100]) / sum(s)
0.60941954345568206
>>> sum(s[:200]) / sum(s)
0.81150074754767521
>>> sum(s[:300]) / sum(s)
0.95132368373281095

>>> lsa = numpy.dot(u, numpy.diag(s)[:,:100])
>>> lsa.shape
(382, 100)

>>> from sklearn.cluster import KMeans
>>> kmeans = KMeans(n_clusters=4, random_state=0).fit(lsa)
>>> kmeans.labels_
array(
 [1, 1, 3, 0, 0, 3, 1, 1, 3, 1, 2, 0, 1, 0, 2, 0, 1, 0, 1, 0, 0, 3, 0,
  0, 3, 3, 1, 0, 1, 1, 1, 1, 3, 1, 1, 1, 1, 2, 1, 1, 1, 0, 1, 1, 1, 2,
  0, 0, 2, 1, 1, 1, 0, 3, 3, 3, 0, 1, 1, 2, 0, 1, 3, 0, 1, 2, 1, 0, 0,
  3, 2, 2, 3, 0, 2, 3, 3, 2, 0, 1, 1, 3, 0, 3, 1, 0, 3, 3, 1, 0, 3, 0,
  1, 1, 0, 1, 1, 3, 0, 3, 1, 1, 1, 1, 0, 1, 2, 1, 2, 1, 0, 1, 0, 2, 0,
  0, 0, 1, 1, 1, 2, 1, 3, 3, 2, 3, 0, 2, 3, 0, 1, 0, 1, 2, 1, 3, 0, 2,
  1, 2, 3, 3, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 0, 2,
  0, 2, 0, 3, 2, 3, 2, 1, 3, 1, 2, 2, 2, 1, 3, 1, 3, 3, 3, 3, 0, 1, 2,
  2, 1, 2, 2, 3, 1, 3, 1, 1, 1, 2, 1, 3, 2, 3, 2, 2, 2, 2, 1, 2, 1, 2,
  2, 1, 3, 0, 2, 2, 1, 3, 2, 1, 0, 2, 1, 3, 3, 2, 1, 3, 1, 3, 3, 1, 1,
  2, 0, 1, 0, 3, 3, 0, 3, 3, 2, 1, 1, 2, 3, 2, 3, 3, 2, 3, 3, 1, 2, 2,
  2, 1, 1, 0, 1, 3, 1, 3, 3, 2, 3, 2, 3, 2, 2, 1, 2, 3, 2, 3, 2, 2, 2,
  2, 2, 2, 2, 2, 2, 3, 1, 2, 1, 3, 3, 1, 2, 3, 3, 3, 2, 3, 3, 1, 1, 3,
  2, 3, 2, 3, 1, 2, 1, 2, 2, 0, 2, 2, 3, 1, 1, 2, 2, 2, 2, 2, 0, 1, 1,
  1, 1, 3, 0, 3, 2, 2, 2, 3, 1, 1, 0, 1, 2, 1, 2, 0, 1, 2, 2, 0, 3, 3,
  0, 3, 0, 1, 3, 3, 1, 2, 1, 1, 3, 1, 1, 0, 3, 3, 3, 2, 3, 1, 3, 1, 2,
  1, 2, 3, 1, 2, 3, 0, 2, 0, 3, 0, 1, 1, 3], dtype=int32)
>>> cnt = {}
>>> for label in CATEGORIES:
...    cnt[label] = [0, 0, 0, 0]
>>> for i in range(0, 382):
...   cnt[data[i]['label']][kmeans.labels_[i]] += 1
>>> cnt
{'housing': [0, 7, 4, 9], 'coffee': [38, 55, 19, 27],
 'money-supply': [13, 45, 66, 50], 'rubber': [9, 15, 10, 15]}

>>> cnt
{'housing': [14, 5, 0, 1], 'coffee': [55, 59, 18, 7],
 'money-supply': [113, 41, 9, 11], 'rubber': [16, 24, 9, 0]}

$ unzip stanford-ner-2015-12-09.zip
$ cd stanford-ner-2015-12-09

$ cat sample.txt
The fate of Lehman Brothers, the beleaguered investment bank, hung in
the balance on Sunday as Federal Reserve officials and the leaders of
major financial institutions continued to gather in emergency meetings
trying to complete a plan to rescue the stricken bank.  Several
possible plans emerged from the talks, held at the Federal Reserve
Bank of New York and led by Timothy R. Geithner, the president of the
New York Fed, and Treasury Secretary Henry M. Paulson Jr.

$ java -mx600m -cp 'stanford-ner.jar:lib/*' \
    edu.stanford.nlp.ie.crf.CRFClassifier \
    -loadClassifier classifiers/english.all.3class.distsim.crf.ser.gz \
    -textFile sample.txt
The/O fate/O of/O Lehman/ORGANIZATION Brothers/ORGANIZATION
,/O the/O beleaguered/O investment/O bank/O ,/O hung/O in/O the/O
balance/O on/O Sunday/O as/O Federal/ORGANIZATION
Reserve/ORGANIZATION officials/O and/O the/O leaders/O of/O major/O
financial/O institutions/O continued/O to/O gather/O in/O emergency/O
meetings/O trying/O to/O complete/O a/O plan/O to/O rescue/O the/O
stricken/O bank/O ./O Several/O possible/O plans/O emerged/O from/O
the/O talks/O ,/O held/O at/O the/O Federal/ORGANIZATION
Reserve/ORGANIZATION Bank/ORGANIZATION of/ORGANIZATION
New/ORGANIZATION York/ORGANIZATION and/O led/O by/O
Timothy/PERSON R./PERSON Geithner/PERSON ,/O the/O president/O
of/O the/O New/ORGANIZATION York/ORGANIZATION Fed/ORGANIZATION
,/O and/O Treasury/ORGANIZATION Secretary/O Henry/PERSON
M./PERSON Paulson/PERSON Jr./PERSON ./O

>>> import spacy
>>> nlp = spacy.en.English()
>>> doc = nlp(open('sample.txt').read())
>>> doc.ents
(Lehman Brothers, Sunday, Federal Reserve, the Federal Reserve Bank,
New York, Timothy R. Geithner, the New York Fed, Treasury,
Henry M. Paulson Jr.)

$ git clone https://github.com/knowitall/openie
$ cd openie
$ sbt package
$ sbt 'run-main edu.knowitall.openie.OpenIECli’
> WhatsApp 2.12.9 brings 3D Touch gestures to iPhone 6s: Download now http://t.co/YI7m2vutlR http://t.co/qubfpGuWmo
0.95 (WhatsApp 2.12.9; brings; 3D Touch gestures; to iPhone 6s)
> How Will Record iPhone 6s, 6s Plus Sales Affect Apple, AAPL, Earnings? http://t.co/E8RfDMblmG #Bing
0.94 (Will Record iPhone 6s; Affect; Apple, AAPL, Earnings)

$ echo 'WhatsApp 2.12.9 brings 3D Touch gestures to iPhone 6s: Download now http://t.co/YI7m2vutlR http://t.co/qubfpGuWmo' > /tmp/input.txt

$ echo 'How Will Record iPhone 6s, 6s Plus Sales Affect Apple, AAPL, Earnings? http://t.co/E8RfDMblmG #Bing' >> /tmp/input.txt

$ sbt -J-Xmx2700M clean compile assembly

$ java -Xmx4g -XX:+UseConcMarkSweepGC \
    -jar ./target/scala-2.10/openie-assembly-4.1.4-SNAPSHOT.jar \
    --format column \
    < /tmp/input.txt

$ brew install juman

$ sudo apt-get install juman

$ echo ちょっと日本語分かりますけど。 | juman -b
ちょっと ちょっと ちょっと 副詞 8 * 0 * 0 * 0 "代表表記:ちょっと/ちょっと 相対名詞修飾 用言弱修飾"
日本 にほん 日本 名詞 6 地名 4 * 0 * 0 "代表表記:日本/にほん 地名:国"
語 ご 語 名詞 6 普通名詞 1 * 0 * 0 "代表表記:語/ご 漢字読み:音 カテゴリ:抽象物"
分かり わかり 分かる 動詞 2 * 0 子音動詞ラ行 10 基本連用形 8 "代表表記:分かる/わかる"
ます ます ます 接尾辞 14 動詞性接尾辞 7 動詞性接尾辞ます型 31 基本形 2 "代表表記:ます/ます"
けど けど けど 助詞 9 接続助詞 3 * 0 * 0 NIL
。 。 。 特殊 1 句点 1 * 0 * 0 NIL
EOS

$ echo アルバイトの若い人でしたよ。| juman -b
アルバイト あるばいと アルバイト 名詞 6 サ変名詞 2 * 0 * 0 "代表表記:アルバイト/あるばいと カテゴリ:人;抽象物 ドメイン:ビジネス"
の の の 助詞 9 格助詞 1 * 0 * 0 NIL
若い わかい 若い 形容詞 3 * 0 イ形容詞アウオ段 18 基本形 2 "代表表記:若い/わかい"
人 じん 人 名詞 6 普通名詞 1 * 0 * 0 "代表表記:人/じん 漢字読み:音 カテゴリ:人"
でした でした だ 判定詞 4 * 0 判定詞 25 デス列タ形 30 NIL
よ よ よ 助詞 9 終助詞 4 * 0 * 0 NIL
。 。 。 特殊 1 句点 1 * 0 * 0 NIL
EOS

$ echo 東京へ行った。 | juman -b
東京 とうきょう 東京 名詞 6 地名 4 * 0 * 0 "代表表記:東京/とうきょう 地名:日本:都"
へ へ へ 助詞 9 格助詞 1 * 0 * 0 NIL
行った いった 行く 動詞 2 * 0 子音動詞カ行促音便形 3 タ形 10 "代表表記:行く/いく 付属動詞候補（タ系） ドメイン:交通 反義:動詞:帰る/かえる"
。 。 。 特殊 1 句点 1 * 0 * 0 NIL
EOS

$ echo 鈴木さんは優しい人ですね。 | juman -b
鈴木 すずき 鈴木 名詞 6 人名 5 * 0 * 0 "人名:日本:姓:1:0.00961"
さん さん さん 接尾辞 14 名詞性名詞接尾辞 2 * 0 * 0 "代表表記:さん/さん"
は は は 助詞 9 副助詞 2 * 0 * 0 NIL
優しい やさしい 優しい 形容詞 3 * 0 イ形容詞イ段 19 基本形 2 "代表表記:優しい/やさしい"
人 じん 人 名詞 6 普通名詞 1 * 0 * 0 "代表表記:人/じん 漢字読み:音 カテゴリ:人"
です です だ 判定詞 4 * 0 判定詞 25 デス列基本形 27 NIL
ね ね ね 助詞 9 終助詞 4 * 0 * 0 NIL
。 。 。 特殊 1 句点 1 * 0 * 0 NIL
EOS

$ curl https://nlp.stanford.edu/software/stanford-segmenter-2018-02-27.zip > stanford-segmenter-2018-02-27.zip
$ tar xf stanford-segmenter-2018-02-27.zip
$ cd stanford-segmenter-2018-02-27

$ cat chinese.txt
霍洛威的双亲批评阿鲁巴警方在案件调查及质询最后看到他们女儿的三名男子期间工作有欠严谨，还呼吁杯葛阿鲁巴并得到亚拉巴马州州长鲍勃·赖利支持，但没有获得联邦层面的支持。
2012年1月12日，亚拉巴马州法官宣告娜塔莉·安·霍洛威在法律上已经死亡。

$ ./segment.sh ctb chinese.txt UTF-8 0
霍洛威 的 双亲 批评 阿鲁巴 警方 在 案件 调查 及 质询 最后 看到 他们 女儿 的 三 名 男子 期间 工作 有欠 严谨 ， 还 呼吁 杯葛 阿鲁巴 并 得到 亚拉巴马州 州长 鲍勃·赖利 支持 ， 但 没有 获得 联邦 层面 的 支持 。
2012年 1月 12日 ， 亚拉巴马州 法官 宣告 娜塔莉·安·霍洛威 在 法律 上 已经 死亡 。