data_helpers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "# standard\n",
    "from csv import DictReader\n",
    "from operator import itemgetter\n",
    "import re\n",
    "\n",
    "# extra\n",
    "from funcy import memoize\n",
    "from spacy.lemmatizer import Lemmatizer\n",
    "from spacy.lang.en import LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES\n",
    "from symspellpy.symspellpy import SymSpell, Verbosity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WARNING: words of length <= 4 will not get spelling correction\n",
    "special_spelling = {\n",
    "    'believ': 'believe', 'bbq': 'barbecue', 'cauliflowerets': 'cauliflower', \"chik''n\": 'chicken',\n",
    "    'chillies': 'chile', 'coco': 'cocoa', 'creamer': 'cream', 'cummin': 'cumin', 'dress': 'dressing', 'dri': 'dried',\n",
    "    'evapor': 'evaporated', 'filo': 'phyllo', 'fri': 'fries', 'grained': 'grain',\n",
    "    'haas': 'hass', 'half half': 'halfandhalf', 'ic': 'ice',\n",
    "    'jalape': 'jalapeno', 'jell o': 'jello', 'lb': '', 'leav': 'leaf', 'leaves': 'leaf', 'liquorice': 'licorice', \n",
    "    'marin': 'marinade', 'mayonnai': 'mayonaise', 'min': 'mint', 'minicub': 'minicube',\n",
    "    'nuts': 'nut', 'oats': 'oat', 'oliv': 'olive', 'pâté': 'pate',\n",
    "    'penn': 'penne', 'purée': 'puree', 'dress russian': 'russian dressing', 'rom': 'roma',\n",
    "    'saki': 'sake', 'sauc': 'sauce', 'season': 'seasoning', 'soi': 'soy', 'tradit': 'traditional', 'v 8': 'v8',\n",
    "    'veget': 'vegetable', 'vegeta': 'vegetable', 'veggie': 'vegetable', 'veggies': 'vegetable',\n",
    "    'wrapper': 'wrap', 'wrappers': 'wrap', 'yuca': 'yucca'\n",
    "}\n",
    "\n",
    "exceptions = {\n",
    "    # commas\n",
    "    'boneless, skinless chicken breast': 'boneless skinless chicken breast', 'red vinegar white white, wine,': 'red vinegar and white wine',\n",
    "    # dashes\n",
    "    'all-purpose': 'allpurpose', 'bone-in': 'bonein', 'cho-cho': 'chocho', 'coca-cola': 'coke',\n",
    "    'corn-on-the-cob': 'corn', 'demi-glace': 'demiglace', 'e-fu': 'yi mein', 'five-spice': 'fivespice',\n",
    "    'free-range': 'freerange', 'grass-fed': 'grassfed', 'half-and-half': 'halfandhalf', 'jell-o': 'jello',\n",
    "    'multi-grain': 'multigrain', 'ogura-an': 'oguraan', 'piri-piri': 'piripiri',\n",
    "    'ro-tel': 'rotel', 'shell-on': 'shellon', 'skin-on': 'skinon', 't-bone': 'tbone', 'tex-mex': 'texmex',\n",
    "    'wish-bone': 'wishbone', 'yaki-nori': 'yakinori',\n",
    "    # special\n",
    "    'abura age': 'aburaage', 'bake soda': 'bakingsoda', 'baking soda': 'bakingsoda', 'bun': 'roll', 'buns': 'roll',\n",
    "    'color': 'coloring', 'di parma': '', 'free range': 'freerange',\n",
    "    'garlic clove': 'garlic', 'garlic cloves': 'garlic', 'hot dog': 'hotdog', 'hot dogs': 'hotdog', 'ice cream': 'icecream', \n",
    "    'rib eye': 'ribeye', 'slice thinli': '', 'soft drink': 'soda', 'wish bone': 'wishbone', 'or white yellow': '',\n",
    "    # stopwords\n",
    "    'all purpose': 'allpurpose', 'multi purpose': 'multipurpose',\n",
    "    'bone in': 'bonein',\n",
    "    'head on': 'headon', 'shell on': 'shellon', 'skin on': 'skinon', 'on the vine': '',\n",
    "    'baby back': 'babyback',\n",
    "    'packed in': 'in',\n",
    "    'made with': 'with',\n",
    "    'italian with herbs dressing': 'italian herbs dressing',\n",
    "    'cold cut': 'coldcut', 'cold cuts': 'coldcuts',\n",
    "    'do chua': 'pickled carrot daikon',\n",
    "    'ear of': '', 'leg of': '', 'breast of': '', 'eye of ': 'eyeof', 'fillet of': '', 'rack of': '',\n",
    "    'a touch of': '', 'a hint of': '',\n",
    "    'five spice': 'fivespice',\n",
    "    'bread and butter': 'breadandbutter', 'half and half': 'halfandhalf', 'macaroni and cheese': 'macaroniandcheese', \n",
    "    'pork and beans': 'porkandbeans', 'sweet and sour': 'sweetandsour'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "brands_to_sub_lists = { \n",
    "    'baking mix': ['bisquick'],\n",
    "    'beer': ['budweiser'],\n",
    "    'bouillon': ['better than bouillon'],\n",
    "    'brandy': ['poire williams'],\n",
    "    'bread': ['wondra'],\n",
    "    'butter': ['country crock calcium plus vitamin d', 'flora buttery',\n",
    "               'i cant believe it not butter', 'i cant believe its not butter'],\n",
    "    'cake': ['yodel'],\n",
    "    'candy': ['m and ms'],\n",
    "    'cereal': ['chex', 'corn flakes', 'fiber one'],\n",
    "    'cheese spread': ['velveeta'],\n",
    "    'chocolate': ['hershey', 'ibarra'],\n",
    "    'chip': ['tostitos'],\n",
    "    'corn': ['green giant steamers niblets', 'green giant steamers niblet'],\n",
    "    'cracker': ['triscuit', 'triscuits'],\n",
    "    'cream': ['elmlea single', ],\n",
    "    'dairy': ['daiya', 'flora cuisine', 'flora original'],\n",
    "    'dressing': ['wishbone'],\n",
    "    'gelatin': ['jello'],\n",
    "    'gravy': ['gravy master'],\n",
    "    'hazelnut spread': ['nutella'],\n",
    "    'hot pepper': ['peppadew', 'peppadews', 'piment despelette'],\n",
    "    'hot sauce': ['franks', 'harissa', 'louisiana hot sauce', 'pickapeppa'],\n",
    "    'liqueur': ['chambord', 'crème de cassis', 'creme de cassis', 'cointreau', 'frangelico', 'galliano',\n",
    "                'gran marnier', 'grand marnier',\n",
    "                'kahlua', 'kahlúa', 'licor 43', 'madeira', 'praline liqueur', 'pernod', 'tuaca'],\n",
    "    'liquor': ['southern comfort'],\n",
    "    'meat substitute': ['quorn'],\n",
    "    'milk': ['carnation'],\n",
    "    'pasta': ['knorr pasta sides', 'barilla'],\n",
    "    'rice': ['uncle bens', 'knorr fiesta sides'],\n",
    "    'rice flour': ['mochiko'],\n",
    "    'rum': ['bacardi'],\n",
    "    'seasoning': ['accent', 'maggi', 'mrs dash', 'taco bell home originals'],\n",
    "    'shortening': ['crisco'],\n",
    "    'soda': ['7 up', 'coke', 'dr pepper', 'la casera', 'sprite', 'squirt'],\n",
    "    'stout': ['guinness'],\n",
    "    'sugar': ['sugar in the raw'],\n",
    "    'sweetener': ['splenda', 'splenda granular', 'splenda granulated', 'swerve', 'truvía', 'truvía baking blend'],\n",
    "    'syrup': ['karo'],\n",
    "    'tequila': ['jose cuervo'],\n",
    "    'tomato': ['rotel'],\n",
    "    'tomato juice': ['v8'],\n",
    "    'tomato sauce': ['ragu'],\n",
    "    'tortilla chip': ['doritos'],\n",
    "    'whiskey': ['george dickel', 'jack daniels'],\n",
    "    'yogurt': ['flora proactiv', 'yoplait']\n",
    "}\n",
    "\n",
    "brands_to_pull = {\n",
    "    'a taste of thai', 'alexia', 'angostura', 'argo', 'azteca', 'baileys', 'barilla ovenready',\n",
    "    'barilla plus', 'bertolli', 'bertolli classico', 'best food', 'best foods', 'betty crocker', 'bob evans',\n",
    "    'bragg', 'braggs', 'breakstones', 'breyers', 'camellia', 'campbells', 'cavenders', 'challenge',\n",
    "    'chartreuse', 'chobani', 'cholula', 'cinnamon toast crunch', 'colmans', 'conimex wok',\n",
    "    'conimex woksaus specials', 'country crock', 'crisco', 'crisco pure', 'crystal', 'crystal farms', 'cumberland',\n",
    "    'curry guy', 'daisy', 'daisy brand', 'del monte', 'delallo', 'diamond crystal', 'dole', 'domaine de canton',\n",
    "    'domino', 'dream', 'dreamfields', 'duncan hines', 'earth balance', 'egglands best', 'equal', 'estancia',\n",
    "    'everglades', 'fisher', 'foster farms', 'franks', 'franks redhot original', 'frenchs', \n",
    "    'gebhardt', 'godiva', 'gold medal', 'good seasons', 'gourmet garden', 'goya', 'green giant',\n",
    "    'grey poupon', 'hatch', 'heath', 'heinz', 'hellmann', 'hellmanns', 'herdez',\n",
    "    'hidden valley', 'hidden valley farmhouse originals', 'hogue', 'holland house', 'honey bunches of oat',\n",
    "    'honeysuckle white', 'hurst family harvest', 'imperial',\n",
    "    'jack daniels', 'jagermeister', 'jameson', 'jello', 'jif', 'jiffy', 'jimmy dean', 'johnsonville',\n",
    "    'kahlua', 'kahlúa', 'kerrygold', 'kewpie', 'kikkoman', 'kim crawford', 'king arthur',\n",
    "    'klondike gourmet', 'klondike rose', 'knorr', 'knox', 'knudsen', 'kraft', 'kraft big slice', 'kraft classic',\n",
    "    'kraft mexican style', 'kraft original', 'kraft slim cut', 'kroger', 'la victoria', 'land o lakes',\n",
    "    'lea and perrins', 'lipton', 'lipton cup size', 'lipton iced tea brew family size', 'lipton recipe secrets',\n",
    "    'madras', 'mae ploy', 'makers mark', 'maldon', 'manischewitz', 'martha white', 'mazola', 'mccormick',\n",
    "    'mccormick original', 'mccormick perfect pinch', 'meyer', 'mezzetta', 'minute',\n",
    "    'mission', 'mizkan', 'morton', 'mountain dew', 'mountain high',\n",
    "    'nakano', 'nestle', 'new york style panetini', 'nido', 'nielsenmassey', 'nielsen massey', 'nilla', 'nusalt', 'nu salt',\n",
    "    'old bay', 'old el paso',\n",
    "    'old el paso thick n chunky', 'oreo', 'ortega', 'oscar mayer', 'oscar mayer deli fresh', 'pace', 'pam', \n",
    "    'pam nostick', 'pepperidge farm', 'philadelphia', 'pillsbury', 'pillsbury classic', \n",
    "    'pillsbury crescent recipe creations', 'pompeian', 'progresso', 'pure wesson', 'ragu classic',\n",
    "    'ragu robusto', 'ragu traditional', 'red gold', 'rice krispies', 'ritz', 'robert mondavi', 'ronzoni',\n",
    "    'royal', 'saffron road', 'sargento', 'sargento artisan blends', 'sargento traditional cut', 'silk', 'simply organic',\n",
    "    'skippy', 'smart balance', 'smithfield', 'southern comfort', 'soy vay', 'soy vay veri veri teriyaki', 'special k',\n",
    "    'spice islands', 'spike', 'st germain', 'stonefire', 'success', 'swanson',\n",
    "    'syd', 'tabasco', 'taco bell', 'taco bell thick and chunky', 'tapatio', 'texas pete',\n",
    "    'thai kitchen', 'tipo ', 'toulouse', 'tuttorosso',\n",
    "    'tyson', 'uncle bens original converted brand', 'uncle bens ready rice', 'velveeta', 'white lily',\n",
    "    'wholesome sweeteners', 'wolf brand', 'yoplait', 'zatarains'   \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_to_segment = {\n",
    "    'alfredostyle': 'alfredo style', 'almondmilk': 'almond milk', 'applesauce': 'apple sauce',\n",
    "    'arrowroot': 'arrow root', 'beetroot': 'beet root',\n",
    "    'bellpepper': 'bell pepper', 'blackcurrant': 'black currant', 'blackpepper': 'black pepper', 'bluefish': 'blue fish',\n",
    "    'breadcrumb': 'bread crumb', 'breadcrumbs': 'bread crumbs', 'breadstick': 'bread stick', 'brinecured': 'brine cured',\n",
    "    'broilerfryer': 'broiler fryer', 'broilerfryers': 'broiler fryer', 'butterflavored': 'butter flavored',\n",
    "    'buttermargarine': 'butter margarine', 'chickenapple': 'chicken apple', 'chilegarlic': 'chile garlic', \n",
    "    'chocolatecovered': 'chocolate covered', 'chocolatehazelnut': 'chocolate hazelnut', 'coarsegrain': 'coarse grain',\n",
    "    'coconutmilk': 'coconut milk', 'countrystyle': 'country style', 'cornbread': 'corn bread', 'corncobs': 'corn cobs', \n",
    "    'cornflour': 'corn flour', 'cornmeal': 'corn meal', 'cornstarch': 'corn starch',\n",
    "    'crabapples': 'crab apples', 'cuminseed': 'cumin seed', 'dillweed': 'dill weed', 'dutchprocessed': 'dutch processed',\n",
    "    'extralean': 'extra lean', 'extravirgin': 'extra virgin', 'flaxseed': 'flax seed', 'freezedried': 'freeze dried',\n",
    "    'fruitcake': 'fruit cake', 'gingerroot': 'ginger root', 'greekstyle': 'greek style', \n",
    "    'hardboiled': 'hard boiled', 'honeyflavored': 'honey flavored', 'italianstyle': 'italian style',\n",
    "    'kiwifruit': 'kiwi fruit', 'kongstyle': 'kong style', 'lemonlime': 'lemon lime', 'longgrain': 'long grain',\n",
    "    'meatfilled': 'meat filled', 'mediumgrain': 'medium grain', 'ovenready': 'oven ready',\n",
    "    'parmigianareggiano': 'parmigiana reggiano', 'parmigianoreggiano': 'parmigiano reggiano', 'pinenut': 'pine nut',\n",
    "    'poppyseeds': 'poppy seeds', 'poundcake': 'pound cake', 'pumpkinseed': 'pumpkin seed', 'pumpkinseeds': 'pumpkin seed',\n",
    "    'quickcooking': 'quick cooking', 'redcurrant': 'red currant', 'sesameginger': 'sesame ginger', 'shanghaistyle': 'shanghai style',\n",
    "    'sheepshead': 'sheeps head', 'shiromiso': 'shiro miso', 'shortgrain': 'short grain', 'softboiled': 'soft boiled',\n",
    "    'soybean': 'soy bean', 'soybeans': 'soy bean', 'soymilk': 'soy milk', 'sparerib': 'spare rib', 'stoneground': 'stone ground',\n",
    "    'sugarcane': 'sugar cane', 'superfine': 'super fine', \n",
    "    'sweetbreads': 'sweet breads', 'vegetablefilled': 'vegetable filled', 'wholemilk': 'whole milk'\n",
    "}\n",
    "\n",
    "state_words = {\n",
    "    # prep\n",
    "    'beaten', 'blackened', 'blanched', 'bleached', 'blend', 'blended', 'boiled', 'bonein', 'boneles',\n",
    "    'bottled', 'braised', 'breaded', 'brewed', \n",
    "    'candied', 'canned', 'centercut', 'chiffonade', 'chilled', 'chopped',\n",
    "    'clotted', 'coarse', 'cold', 'coldsmoked', 'compressed',\n",
    "    'condensed', 'cook', 'cooked', 'cool', 'cracked', 'creamed', 'crumble', 'crumbled', 'crushed', 'cube',\n",
    "    'cubed', 'cured', 'cut', 'diced', 'drained', 'dried', 'flaked', 'fleshed', 'fried', 'frozen',\n",
    "    'granular', 'granulate', 'granulated', 'grate', 'grated', 'grill', 'grilled', 'ground', 'headon', \n",
    "    'iced', 'iodized', 'mashed', 'melted', 'minced', 'mixed', 'natural', 'parboiled', 'pack', 'packed',\n",
    "    'peeled', 'pickled', 'pitted', 'pointed', 'popped',\n",
    "    'powdered', 'prepare', 'prepared', 'pressed', 'raw', 'reduce', 'reduced', 'refined',\n",
    "    'refrigerated', 'ripe', 'ripened', 'roasted',\n",
    "    'rolled', 'rubbed', 'salted', 'section', 'segment', 'shaped', 'shaved', 'shelled', 'shellon', 'shred', 'shredded',\n",
    "    'shuck', 'shucked', 'skinles', 'skinon', 'slice', 'sliced',\n",
    "    'slivered', 'smoked', 'softened','steamed', 'stewed', 'strained', 'sweeten', 'sweetened', 'toasted',\n",
    "    'uncook', 'unsalt', 'unbleached', 'uncooked', 'unsalted', 'unsmoked', 'unsulphured',\n",
    "    'warm', 'whipped', 'whisked', 'whole',\n",
    "    # quality\n",
    "    'added', 'allpurpose', 'artificial', 'best', 'blend', 'bought',\n",
    "    'crispy', 'dry', 'extra', 'fancy', 'fine', 'finely', 'firm', 'firmli', 'firmly', 'flavor',\n",
    "    'flavored', 'freerange', 'fresh', 'freshly', 'fully', 'grassfed', 'homemade', 'imitation', 'large',\n",
    "    'leftover', 'loosely', 'natural', 'medium', 'mini',\n",
    "    'multipurpose', 'oldfashioned', 'organic', 'petite', 'plain', 'prime', 'pure', \n",
    "    'regular', 'seasoned', 'seedless', 'size', 'small', 'spare', 'style', 'super', 'superior',\n",
    "    'table', 'tightly', 'tradicional', 'traditional', 'unflavored', 'virgin',\n",
    "}\n",
    "\n",
    "heads_to_drop = {\n",
    "    'accompaniment', 'bag', 'baking', 'ball', 'bar', 'base', 'black', 'bowl',  'cap',\n",
    "    'crosswise',  'cube', 'cup', 'dessert', 'dinner', 'fillet', 'flake', 'food', 'free', 'frozen', \n",
    "    'half', 'kiss', 'layer', 'lowfat', 'mixers', 'pad', 'pod', 'pot', 'purpose', 'quick',\n",
    "    'recipe', 'shape', 'side', 'spanish', 'spiral', 'split', 'sprig', 'steak', 'weed', 'zero'\n",
    "}\n",
    "\n",
    "head_subs = {\n",
    "    'meat': ['fillet', 'steak']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "supertypes = {\n",
    "    'apple': ['pippin'],\n",
    "    'bakingsoda': ['bicarbonate'],\n",
    "    'ball': ['falafel'],\n",
    "    'bean': ['cannelloni', 'garbanzo', 'legume'],\n",
    "    'beer': ['ale', 'lambic', 'pilsner', 'porter', 'stout'],\n",
    "    'bouillon': ['minicube'],\n",
    "    'bread': ['bagel', 'baguette',  'brioche', 'bun', 'ciabatta', 'grissini',\n",
    "             'pumpernickel', 'roll', 'sourdough'],\n",
    "    'butter': ['oleo', 'margarine', 'shortening'],\n",
    "    'cake': ['crepe', 'cupcake', 'genoise', 'pancake', 'shortcake', 'waffle'],\n",
    "    'candy': ['brittle', 'dragee', 'gumdrop', 'meringue', 'praline', 'toffee'],\n",
    "    'cereal': ['amaranth', 'barley', 'buckwheat', 'bran', 'bulgur', 'groat', 'malt', 'millet', 'oat', 'oatmeal',\n",
    "               'sorghum', 'spelt', 'triticale', 'wheat', 'wheatberry', 'wheatberrie'],\n",
    "    'cheese': ['asiago', 'boursin', 'brie', 'cantal', 'cheddar', 'chevre', 'colby', 'dolce', 'dubliner', 'feta', \n",
    "               'fontina', 'gorgonzola', 'gouda', 'gruyere', 'gruyère', 'halloumi', 'havarti', 'jack',\n",
    "               'jarlsberg', 'manchego', 'mascarpone', 'mozzarella', 'muenster', 'neufchtel',\n",
    "               'paneer', 'parmesan', 'parmigiano', 'pecorino', 'provolone', \n",
    "               'reblochon', 'reggiano', 'ricotta', 'romano', 'roquefort',\n",
    "               'stilton', 'swis', 'taleggio', 'wensleydale'],\n",
    "    'cherry': ['maraschino'],\n",
    "    'coffee': ['cappuccino'],\n",
    "    'cookie': ['amaretti', 'biscotti', 'cone', 'gingersnap', 'macaron', 'shortbread', 'snap', 'tart'],\n",
    "    'cracker': ['biscuit', 'pretzel', 'saltine'],\n",
    "    'custard': ['flan'],\n",
    "    'dressing': ['russian', 'vinaigrette'], \n",
    "    'drink': ['beverage', 'carbonated', 'eggnog', 'limeade', 'seltzer', 'tonic', 'verjuice', 'zabaglione'],\n",
    "    'egg': ['white', 'yolk'],\n",
    "    'fish': ['abalone', 'ahi', 'albacore', 'amberjack', 'anchovy', 'barramundi', 'bas', 'basa', 'branzino', 'bream', 'brill',\n",
    "             'carp', 'catfish', 'claw', 'cod' 'conch', 'cuttlefish', 'dogfish', 'eel', 'escargot', 'fishcake', 'grouper', \n",
    "             'haddock', 'hake', 'halibut', 'hamachi', 'herring', 'kamaboko',\n",
    "             'kingfish', 'kipper', 'lingcod', 'mackerel', 'mahi', 'mahimahi', 'merluza', 'milkfish', 'monkfish', 'mullet',\n",
    "             'octopu', 'perch', 'pike', 'pollock', 'pompano', 'octopus', 'redfish', 'rockfish', 'roughy', 'sablefish', 'salmon',\n",
    "             'sashimi', 'scrod', 'shad', 'shark', 'shellfish', 'snapper', 'sockeye', 'sole', 'sturgeon', 'swordfish',\n",
    "             'tentacle', 'tilapia', 'tobiko', 'trout',\n",
    "             'tuna', 'turbot', 'turtle', 'unagi', 'whitefish', 'yellowfin', 'yellowtail'],\n",
    "    'flatbread': ['appam', 'arepa', 'bakarkhani', 'bammy', 'bannock', 'barbari', 'bataw', 'bazlama', 'beiju', 'bhakri', 'bhatura',\n",
    "                  'bindaeddeok', 'bolani', 'borlengo', 'bánh', 'casabe', 'chapati', 'chepati', 'chikkolee', 'dhebra', 'dosa',\n",
    "                  'farl', 'flatbrød', 'flatkaka', 'focaccia', 'frybread', 'ftira', 'gurassa', 'gözleme', 'harsha', 'hoggan',\n",
    "                  'injera', 'johnnycake', 'kaak', 'kachori', 'khebz', 'khubz', 'kulcha', 'lagana', 'lahoh', 'laobing', 'lavash', 'lefse',\n",
    "                  'lepyoshka', 'luchi', 'malooga', 'markook', 'matnakash', 'matzah', 'matzo', 'murr', 'muufo', 'naan', 'ngome', 'oatcake',\n",
    "                  'obi non', 'opłatek', 'paan', 'pappadam', 'paratha', 'parotta', 'pathiri', 'pesarattu', 'phulka', 'piadina', 'piaya',\n",
    "                  'pissaladière', 'pita', 'podpłomyk', 'pogača', 'poli', 'posúch', 'pupusa', 'puri', 'părlenka', 'rieska', 'roti', 'rotti',\n",
    "                  'sanchuisanda', 'sangak', 'semita', 'sheermal', 'shelpek', 'taftan', 'tigella', 'torta', 'tortilla', 'tunnbröd', 'yufka'],\n",
    "    'flour': ['besan'],\n",
    "    'flower': ['dandelion', 'hibiscu', 'hop', 'orchid', 'pansy', 'salsify', 'violet'],\n",
    "    'fruit': ['boysenberry', 'breadfruit', 'citru', 'honeydew', 'jujube', 'kiwi', 'papaya', 'umeboshi', 'wolfberry', 'yuzu'],\n",
    "    'gelatin': ['aspic'],\n",
    "    'honey': ['abbamele'],\n",
    "    'icecream': ['pop', 'popsicle', 'sherbet', 'sorbet', 'mochi'],\n",
    "    'lettuce': ['iceberg'],\n",
    "    'liqueur': ['bénédictine', 'cordial', 'kirschenliqueur', 'kirschwasser', 'schnapp'], \n",
    "    'liquor': ['absinthe', 'alcohol', 'grappa', 'mezcal', 'moonshine'],   \n",
    "    'meat': ['asada', 'bacon', 'bear', 'beef', 'belly', 'bison', 'blade', 'blood', 'boar', 'bone', 'bonein',\n",
    "             'brain', 'breast',\n",
    "             'buffalo', 'burger', 'carne', 'carnita', 'char', 'chateaubriand', 'chicken', 'chop', 'chopmeat', \n",
    "             'chuck', 'coldcut', 'confit', 'cutlet', 'drummette', 'drumstick', 'duck', 'duckling',\n",
    "             'eye', 'eyeofround', 'flank', 'foot', 'fowl', 'fryer',\n",
    "             'game', 'giblet', 'gizzard', 'goat', 'goose', 'ham', 'hanger', 'hen', 'hock', 'iron', \n",
    "             'jerky', 'jowl', 'knuckle', 'lamb', 'lean', 'leg', 'liver', \n",
    "             'loin', 'marrow', 'meatball', 'mignon', 'mincemeat', 'moose', 'mutton', 'neck', 'nugget', 'oxtail', \n",
    "             'pheasant', 'pig', 'pork', 'porterhouse', 'poultry', 'pâté', \n",
    "             'quail', 'rabbit', 'rack', 'rib', 'riblet', 'roast', 'rooster', 'round', 'rump',\n",
    "             'scallopini', 'shank', 'shoulder', 'sirloin', 'skirt', 'spam', 'squirrel', 'strip', 'suet',\n",
    "             'tallow', 'tbone', 'tender', 'tenderloin', 'thigh',\n",
    "             'tongue', 'topside', 'tripe', 'tritip', 'turkey', 'venison', 'wing', 'wingette'],\n",
    "    'mushroom': ['cremini', 'maitake', 'matsutake', 'morel', 'porcini', 'portabello',\n",
    "                 'shiitake', 'shimeji', 'truffle'],\n",
    "    'mustard': ['dijon'],\n",
    "    'noodle': ['capellini', 'fettuccine', 'hair', 'linguine', 'perciatelli', 'raman',\n",
    "               'spaghetti', 'spaghettini', 'soba', 'vermicelli', 'udon'],\n",
    "    'nut': ['acorn', 'almond', 'candlenut', 'cashew', 'chestnut', 'hazelnut',\n",
    "            'macadamia', 'mongongo', 'peanut', 'pecan', 'pistachio', 'walnut'],\n",
    "    'olive': ['kalamata', 'manzanilla'],\n",
    "    'onion': ['vidalia'],\n",
    "    'pasta': ['bowtie', 'cavatappi', 'cavatelli', 'corkscrew', 'ditalini', 'elbow', \n",
    "              'farfallini', 'fideo', 'fusilli', 'gemelli', 'macaroni', \n",
    "              'manicotti', 'orzo', 'pappardelle', 'penne', 'rigate', 'rotelle', 'rotini',\n",
    "              'tagliatelle', 'tortellini', 'tubettini', 'wheel', 'ziti'],\n",
    "    'paste': ['doubanjiang', 'duxelle', 'gochujang', 'membrillo',\n",
    "              'tahini','ulek', 'wasabi', 'yuzukosho'], \n",
    "    'pastry': ['croissant', 'crumpet', 'doughnut', 'muffin', 'scone'],\n",
    "    'pepper': ['aleppo', 'anaheim', 'ancho', 'arbol', 'bell', 'capsicum', 'cascabel', 'cayenne', 'chile', \n",
    "               'chilcostle', 'chipotle', 'fresno', 'guajillo', \n",
    "               'habanero', 'jalapeno', 'mulato', 'piquillo',\n",
    "               'padron', 'pasilla', 'peperoncini', 'pimento', 'poblano',  'serrano', 'shishito'],    \n",
    "    'potato': ['fingerling', 'gold', 'obrien', 'russet', 'yukon'],\n",
    "    'rice': ['arborio', 'basmati', 'carnaroli', 'paella', 'poha', 'risotto'],\n",
    "    'root': ['cassava', 'ginger', 'yucca'],\n",
    "    'salsa': ['verde'],\n",
    "    'sauce': ['adobo', 'aioli', 'alfredo', 'bechamel', 'chimichurri', 'demiglace', 'hoisin', \n",
    "              'glacé', 'hollandaise', 'marinara', 'mole',\n",
    "              'passata', 'pati', 'pesto', 'remoulade', 'roux', 'sriracha', 'sweetandsour',\n",
    "              'tamari', 'tartar', 'teriyaki', 'tzatziki', 'worcestershire'],\n",
    "    'sausage': ['andouille', 'bologna', 'brat', 'bratwurst', 'capicola', 'chorizo', 'frank', 'frankfurter', 'hotdog',\n",
    "                'kielbasa', 'knockwurst', 'link', 'liverwurst', 'merguez', 'pastrami', 'patty', 'salami', 'wiener'],\n",
    "    'seasoning': ['ajwain', 'allspice', 'amchur', 'anise', 'annatto', 'asafoetida', 'bark', 'basil', 'berry',\n",
    "                  'caraway', 'cardamom', 'chervil', 'chia', 'chicory',\n",
    "                  'cilantro', 'cinnamon', 'clove', 'coriander', 'cumin', 'dill', 'extract',\n",
    "                  'flavor', 'flavoring', 'flax', 'furikake', 'grainsofparadise',\n",
    "                  'herb', 'jamaican', 'jerk', 'leaf',\n",
    "                  'lemongras', 'licorice', 'menthe', 'mint', 'mitsuba', 'msg',\n",
    "                  'nigella', 'nutmeg', 'oiloforange', 'oregano', 'paprika', 'parsley', 'pepita', 'peppermint',\n",
    "                  'pimenton', 'pollen', 'poppy', 'powder', 'root', 'rosemary', 'rub', 'saffron', 'sage', 'seed', 'sesame',\n",
    "                  'sorrel', 'spice', 'tarragon', 'tea', 'thread', 'thyme', 'togarashi',\n",
    "                  'zaatar', 'zest', 'épice'],\n",
    "    'seaweed': ['agar ', 'algae', 'arame', 'badderlock', 'bladderwrack', 'carola', 'cava', 'channelled wrack ',\n",
    "                'chlorella', 'cochayuyo ', 'corticata', 'cottonii', 'dulse', 'edulis', 'eucheuma', 'fucales',\n",
    "                'gelidiella', 'gim', 'gutweed', 'hijiki', 'hiromi', 'hiziki', 'hypnea', 'kelp', 'kombu', 'konbu',\n",
    "                'laverbread ', 'moss', 'mozuku', 'nori', 'oarweed', 'ogonori', 'papillatus', 'sargassum', 'spinosum',\n",
    "                'thongweed', 'wakame', 'wrack'],\n",
    "    'shoot': ['bamboo'],\n",
    "    'soda': ['colon'], # colon = get_lemma(cola)\n",
    "    'squash': ['calabaza', 'mirliton', 'qua'],\n",
    "    'sugar': ['turbinado'],\n",
    "    'sweetener': ['erythritol', 'fructose', 'glucose', 'maltose', 'stevia', 'treacle'],\n",
    "    'tea': ['assam', 'ginseng', 'lapsang', 'matcha', 'souchong'],\n",
    "    'tofu': ['aburaage', 'seitan', 'yuba'],\n",
    "    'vegetable': ['cob', 'fern', 'peapod'], \n",
    "    'wine': ['beaujolai', 'blanc', 'burgundy', 'cabernet', 'chardonnay', 'chianti', 'grigio', 'malbec', 'merlot',\n",
    "             'moscato', 'muscadet', 'muscat', 'noir', 'prosecco', 'riesling', 'sangiovese', 'sangria',\n",
    "             'sauvignon', 'shaoxing', 'shiraz', 'vermouth', 'zinfandel'],\n",
    "    'yeast': ['poolish', 'rennet', 'starter']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = { # from spacy, minus a few manually\n",
    "    'a', 'about', 'above', 'across', 'after', 'afterwards', 'again', 'against', 'all', 'almost', 'alone', 'along',\n",
    "    'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'amount', 'an', 'and', 'another', 'any', \n",
    "    'anyhow', 'anyone', 'anything', 'anyway', 'anywhere', 'are', 'around', 'as', 'at', 'back', 'be', 'became',\n",
    "    'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'below', 'beside',\n",
    "    'besides', 'between', 'beyond', 'both', 'bottom', 'but', 'by', 'ca', 'call', 'can', 'cannot', 'could', 'did', 'do',\n",
    "    'does', 'doing', 'done', 'down', 'due', 'during', 'each', 'eight', 'either', 'eleven', 'else', 'elsewhere', 'empty',\n",
    "    'enough', 'even', 'ever', 'every', 'everyone', 'everything', 'everywhere', 'except', 'few', 'fifteen', 'fifty', 'first',\n",
    "    'five', 'for', 'former', 'formerly', 'forty', 'four', 'from', 'front', 'full', 'further', 'get', 'give', 'go',\n",
    "    'had', 'has', 'have', 'he', 'hence', 'her', 'here', 'hereafter', 'hereby', 'herein', 'hereupon', \n",
    "    'hers', 'herself', 'him', 'himself', 'his', 'how', 'however', 'hundred', 'i', 'if', 'in', 'indeed', \n",
    "    'into', 'is', 'it', 'its', 'itself', 'just', 'keep', 'last', 'latter', 'latterly', 'least', 'made',\n",
    "    'make', 'many', 'may', 'me', 'meanwhile', 'might', 'mine', 'more', 'moreover', 'most', 'mostly', 'move',\n",
    "    'much', 'must', 'my', 'myself', 'name', 'namely', 'neither', 'never', 'nevertheless', 'next', 'nine', 'nobody',\n",
    "    'none', 'noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of', 'off', 'often', 'on', 'once', 'one', 'only',\n",
    "    'onto', 'or', 'other', 'others', 'otherwise', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'part', 'per',\n",
    "    'perhaps', 'please', 'put', 'quite', 'rather', 're', 'really', 'regarding', 'same', 'say', 'see', 'seem',\n",
    "    'seemed', 'seeming', 'seems', 'serious', 'several', 'she', 'should', 'show', 'side', 'since', 'six', 'sixty',\n",
    "    'so', 'some', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhere', 'still', 'such', 'take',\n",
    "    'ten', 'than', 'that', 'the', 'their', 'them', 'themselves', 'then', 'thence', 'there', 'thereafter', 'thereby',\n",
    "    'therefore', 'therein', 'thereupon', 'these', 'they', 'third', 'this', 'those', 'though', 'three', 'through',\n",
    "    'throughout', 'thru', 'thus', 'to', 'together', 'too', 'top', 'toward', 'towards', 'twelve', 'twenty', 'two',\n",
    "    'under', 'unless', 'until', 'up', 'upon', 'us', 'used', 'using', 'various', 'very', 'via', 'was', 'we', 'well',\n",
    "    'were', 'what', 'whatever', 'when', 'whence', 'whenever', 'where', 'whereafter', 'whereas', 'whereby', 'wherein',\n",
    "    'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', 'whoever', 'whole', 'whom', 'whose',\n",
    "    'why', 'will', 'with', 'within', 'without', 'would', 'yet', 'you', 'your', 'yours', 'yourself', 'yourselves'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invert dict of {k1: v1, k2: v1, k3: v2} to {v1: [k1, k2], v2: k3}\n",
    "def invert_dict_singles(orig):\n",
    "    new = {}\n",
    "    for k, v in orig.items():\n",
    "        new[v] = new.get(v, [])\n",
    "        new[v].append(k)\n",
    "    return new\n",
    "\n",
    "# invert dict of {k1: [v1, v2], k2: [v3]} to {v1: k1, v2: k1, v3: k2}\n",
    "def invert_dict_lists(orig):\n",
    "    new = {}\n",
    "    for k, vals in orig.items():\n",
    "        for v in vals:\n",
    "            new[v] = k\n",
    "    return new\n",
    "\n",
    "# remove first dupes and keep order: [3, 1, 3, 2, 3] => [1, 2, 3]\n",
    "def remove_first_dupes(lst):\n",
    "    seen = set()\n",
    "    seen_add = seen.add\n",
    "    res = [x for x in reversed(lst) if not (x in seen or seen_add(x))]\n",
    "    res = res[::-1]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# phrase regex cleaning\n",
    "char_pattern = re.compile(r'[®™’%!/\\-\\'\\.\\(\\)]')\n",
    "brand_char_pattern = re.compile(r'\\d+')\n",
    "parenthetical_pattern = re.compile(r'\\(.*\\)')\n",
    "of_pattern = re.compile(r'(\\w) of (?:the )?')\n",
    "low_pattern = re.compile(r'\\b(?:less|light|low|no|non|reduced) ')\n",
    "free_pattern = re.compile(r' free\\b')\n",
    "high_pattern = re.compile(r'\\b(?:full|high) ')\n",
    "heads_to_sub = invert_dict_lists(head_subs)\n",
    "supertype_appends = invert_dict_lists(supertypes)\n",
    "brands_to_sub = invert_dict_lists(brands_to_sub_lists)\n",
    "# WARNING: remove brands in reverse alphabetical order\n",
    "brand_names = sorted(brands_to_pull | set(brands_to_sub.keys()), reverse=True)\n",
    "compiled_brands = re.compile(r'(\\b' + r'\\b)|(\\b'.join(brand_names) + r'\\b)')\n",
    "compiled_spelling = []\n",
    "for k, v in sorted(special_spelling.items()):\n",
    "    k = re.compile(r'(\\b)' + k + r'(\\b)')\n",
    "    v = r'\\1' + v + r'\\2'\n",
    "    compiled_spelling.append((k, v))\n",
    "compiled_exceptions = []\n",
    "for k, v in sorted(exceptions.items()):\n",
    "    k = re.compile(r'(\\b)' + k + r'(\\b)')\n",
    "    v = r'\\1' + v + r'\\2'\n",
    "    compiled_exceptions.append((k, v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# detected language strings to add as ingredients\n",
    "lang_trans = {}\n",
    "with open('data/lang_tags.csv', 'r', encoding='utf-8') as file:\n",
    "    reader = DictReader(file, fieldnames=['word', 'lang'])\n",
    "    for row in reader:\n",
    "        lang_trans[row['word']] = row['lang']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sym_spell = SymSpell(83000, 1, 7)\n",
    "dictionary_path = 'data/frequency_dictionary.txt'\n",
    "if not sym_spell.load_dictionary(dictionary_path, 0, 1):\n",
    "    print(\"Dictionary file not found\")\n",
    "\n",
    "def correct_spelling(word):\n",
    "    suggestions = sym_spell.lookup(word, Verbosity.TOP, 1)\n",
    "    if not suggestions:\n",
    "        return word\n",
    "    return suggestions[0].term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatize = Lemmatizer(LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES)\n",
    "\n",
    "@memoize\n",
    "def get_lemma(word):\n",
    "    return lemmatize(word, u'NOUN')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(ings):\n",
    "    flat = []\n",
    "    for ing in ings:\n",
    "        flat.extend(ing.mods + ing.brands + ing.langs)\n",
    "    return flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WARNING: don't overwrite the existing manually tweaked dictionary!!!\n",
    "def make_freq_dict(word_counts):\n",
    "    lines = []\n",
    "    for word, freq in sorted(word_counts.items(), key=itemgetter(1), reverse=True):\n",
    "        if freq > 4 and len(word) > 3:\n",
    "            lines.append('{0} {1}\\n'.format(word, freq))\n",
    "    with open('data/new_frequency_dictionary.txt', 'w+', encoding='utf-8') as file:\n",
    "        file.write(lines)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}