{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Jpred Demo\n",
    "\n",
    "This demo shows how to create and query a Jpred dataset.\n",
    "\n",
    "![Jpred](./figures/Jpred.png)\n",
    "\n",
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkConf, SparkContext, SQLContext\n",
    "from mmtfPyspark.datasets import jpredDataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = SparkConf().setMaster(\"local[*]\") \\\n",
    "                      .setAppName(\"JpredDemo\")\n",
    "sc = SparkContext(conf = conf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Jpred Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = jpredDataset.get_dataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Display Jpred Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+\n",
      "|scopID |sequence                                                                                                                                                                                                                                                                                                                                                                                                                   |secondaryStructure                                                                                                                                                                                                                                                                                                                                                                                                         |trained|\n",
      "+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+\n",
      "|d2nnua_|SMETLCQRLNVCQDKILTHYENDSTDLRDHIDYWKHMRLECAIYYKAREMGFKHINHQVVPTLAVSKNKALQAIELQLTLETIYNSQYSNEKWTLQDVSLEVYLTAPTGCIKKHGYTVEVQFDGDICNTMHYTNWTHIYICEEASVTVVEGQVDYYGLYYVHEGIRTYFVQFKDDAEKYSKNKVWEVHAGGQVILCPTSVF                                                                                                                                                                                                                  |CHHHHHHHHHHHHHHHHHHHHHCCCCHHHHHHHHHHHHHHHHHHHHHHHCCCCEECCEECCCHHHHHHHHHHHHHHHHHHHHHHCCCCCCCCCCCCCCCHHHHHCCCCCCEEEEEEEEEEEECCCCCCEEEEEEEEEEEEEECCEEEEECCEECCCEEEEEECCEEEEEEEHHHHHHHHCCCCCEEEECCCCEECCCCCCC                                                                                                                                                                                                                  |true   |\n",
      "|d1p9ya_|MQVSVETTQGLGRRVTITIAADSIETAVKSELVNVAKKVRIDGLRKGKVPMNIVAQRYGASVRQDVLGDLMSRNFIDAIIKEKINPAGAPTYVPGEYKLGEDFTYSVEFEVYPEVEL                                                                                                                                                                                                                                                                                                      |CEEEEEECCCCEEEEEEEECHHHHHHHHHHHHHHHHCCCCCCCCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCCCCCEEEEEEEECCCCCCCCEEEEEEEECCCCCCC                                                                                                                                                                                                                                                                                                      |true   |\n",
      "|d2a9sa1|MSLFPGDIEELARRIITDFTPLGLMVSTAESCTGGLIAGALTEIAGSSAVVDRGFVTYTNDAKRDMLGVGTETLTTFGAVSRQTALQMAHGALYRSRANFAVAVTGIAGPGGGSAEKPVGLVHLATKARNGNVLHHEMRYGDIGRTEIRLATVRTALEMLIALNQAG                                                                                                                                                                                                                                                    |CCCCCHHHHHHHHHHHHHHHHHCCCEEEEECCCCCHHHHHHCCCCCCCCCEEEEEEECCHHHHHHHHCCCHHHHHHHCCCCHHHHHHHHHHHHHCCCCCEEEEEEECCCCCCCCCCCCCCEEEEEEEECCCCEEEEEEECCCCCHHHHHHHHHHHHHHHHHHHHHCC                                                                                                                                                                                                                                                    |true   |\n",
      "|d1u84a_|GQQLNRLLLEWIGAWDPFGLGKDAYDVEAASVLQAVYETEDARTLAARIQSIYEFAFDEPIPFPHCLKLARRLLELKQAAS                                                                                                                                                                                                                                                                                                                                          |CCHHHHHHHHHHHHHCCCCCCCCCCHHHHHHHHHHHCCCCCHHHHHHHHHHHHHHHHCCCCCHHHHHHHHHHHHHHHHHHC                                                                                                                                                                                                                                                                                                                                          |true   |\n",
      "|d3a02a_|TFTSFQLEELEKAFSRTHYPDVFTREELAMKIGLTEARIQVWFQNRRAKWR                                                                                                                                                                                                                                                                                                                                                                        |CCCHHHHHHHHHHHHHCCCCCHHHHHHHHHHHCCCHHHHHHHHHHHHHHHC                                                                                                                                                                                                                                                                                                                                                                        |false  |\n",
      "|d4ay0a2|KGTPIQFAENLSWKVDGGKLIAENPSPFYMNIGELTFGGKSIPSHYIPPKSTWAFDLLAGARNVSWRIINDQGGLDRLYSKNVT                                                                                                                                                                                                                                                                                                                                       |CCCCCCCCCCCEEEEECCEEEEEECCCCCEEEEEEEECCEECCCCEECCCEEEEEECCCCCCEEEEEEECCCCCECCCEEEECC                                                                                                                                                                                                                                                                                                                                       |false  |\n",
      "|d1ux5a_|KYPRPHKKLKQLHWEKLDCTDNSIWGTGKAEKFADDLYEKGVLADLEKAFAAREIKSLASKRKEDLQKITFLSRDISQQFGINLHMYSSLSVADLVKKILNCDRDFLQTPSVVEFLSKSEIIEVSVNLARNYAPYSTDWEGVRNLEDAKPPEKDPNDLQRADQIYLQLMVNLESYWGSRMRALTVVTSYEREYNELLAKLRKVDKAVSALQESDNLRNVFNVILAVGNFMNDTSKQAQGFKLSTLQRLTFIKDTTNSMTFLNYVEKIVRLNYPSFNDFLSELEPVLDVVKVSIEQLVNDCKDFSQSIVNVERSVEIGNLSDSSKFHPLDKVLIKTLPVLPEARKKGDLLEDEVKLTIMEFESLMHTYGEDSGDKFAKISFFKKFADFINEYKKAQAQNLAAEEEERLYIKH|CCCCCCCCECCCCCCCCCCCCCCCCCCCHHHHHHHHHHHCCHHHHHHHHCECCCCHHHHHHHHHCCCCECCCCHHHHHHHHHHCCCCCCCCHHHHHHHHHCCCHHHHCCHHHHHHCCCHHHHCCCHHHHHHCCCCCCCCCCCCCCCCCCCCCCCCCCECHHHHHHHHCCCCCCCCHHHHHHHHHHHCCHHHHHHHHHHHHHHHHHHHHHHHCCHHHHHHHHHHHHHHHHHCCCCCCCCCCCCCCCCCCCCCECCCCCCEHHHHHHHHHHHHCCCCCCHHHHCHHHHHHCCCCHHHHHHHHHHHHHHHHHHHHHHHHCCCCCCCCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCCC|true   |\n",
      "|d2v89a_|SPEFGYWITCCPTCDVDINTWVPFYSTELNKPAMIYCSHGDGHWVHAQCMDLEERTLIHLSEGSNKYYCNEHVQIARA                                                                                                                                                                                                                                                                                                                                             |CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCEEEECCCCCEEEECCCCCCCHHHHHHHHHCCCCCCCCCCCCCCCC                                                                                                                                                                                                                                                                                                                                             |true   |\n",
      "|d1lm8v_|RPVLRSVNSREPSQVIFCNRSPRVVLPVWLNFDGEPQPYPTLPPGTGRRIHSYRGHLWLFRDAGTHDGLLVNQTELFVPSLNVDGQPIFANITLPVYTLKERCLQVVRSLVKPENYRRLDIVRSLYEDLEDHPNVQKDLERLTQERIAHQ                                                                                                                                                                                                                                                                     |CCCCCCCCCCCEEEEEEEECCCCCEEEEEECCCCCEEECCCECCCEEEEEEEECCCEEEEEECCCCCECEECCECCECCCCCECCECEEEEEECCCCCHHHHHHHHHHHHCCCCCCCCCCCCHHHHHHHHCCCCHHHHHHHHHHHHHCCC                                                                                                                                                                                                                                                                     |false  |\n",
      "|d1w23a_|VKQVFNFNAGPSALPKPALERAQKELLNFNDTQMSVMELSHRSQSYEEVHEQAQNLLRELLQIPNDYQILFLQGGASLQFTMLPMNLLTKGTIGNYVLTGSWSEKALKEAKLLGETHIAASTKANSYQSIPDFSEFQLNENDAYLHITSNNTIYGTQYQNFPEINHAPLIADMSSDILSRPLKVNQFGMIYAGAQKNLGPSGVTVVIVKKDLLNTKVEQVPTMLQYATHIKSDSLYNTPPTFSIYMLRNVLDWIKDLGGAEAIAKQNEEKAKIIYDTIDESNGFYVGHAEKGSRSLMNVTFNLRNEELNQQFLAKAKEQGFVGLNGHRSVGGCRASIYNAVPIDACIALRELMIQFKENA                                                   |CCCCEECCCCCCCCCHHHHHHHHHCCCCCCCCCCCCCCCCCCCHHHHHHHHHHHHHHHHHHCCCCCEEEEEECCHHHHHHHHHHHHHCCCCCEEEEEECCHHHHHHHHHHHCCCEEEEEEECCCCCCCCCCCCCCCCCCCCEEEEEEECEECCCCEECCCCCCCCCCCEEEECCCCCCCCCCCCCCCCEEEEECCCCCCCCCCEEEEEEHHHHCCCCCCCCCCCCHHHHHHCCCCCCCCCHHHHHHHHHHHHHHHHCCHHHHHHHHHHHHHHHHHHHHHCCCCCCEECCCCCCECCCEEEEECCCHHHHHHHHHHHHHCCEECCECCCCCCCEEEECCCCCCHHHHHHHHHHHHHHHHHC                                                   |true   |\n",
      "|d2nmla1|SHTILLVQPTKRPEGRTYADYESVNECMEGVCKMYEEHLKRMNPNSPSITYDISQLFDFIDDLADLSCLVYRADTQTYQPYNKDWIKEKIYVLLRRQAQQ                                                                                                                                                                                                                                                                                                                       |CCEEEEEECCCCCCCCEEEEECCHHHHHHHHHHHHHHHHHHHCCCCCCCCCCHHHHHHHHHHCCEEEEEEEECCCCEEEEECHHHHHHHHHHHHHHHHHC                                                                                                                                                                                                                                                                                                                       |true   |\n",
      "|d4e2va_|RPRFSFSIAAREGKARTGTIEMKRGVIRTPAFMPVGTAATVKALKPETVRATGADIILGNTYHLMLRPGAERIAKLGGLHSFMGWDRPILTDSGGFQVMKQSEEGVTFKSHSRHMLSPERSIEIQHLLGSDIVMAFDEVTPYPATPSRAASSMERSMRWAKRSRDAFDSRKEQAENAALFGIQQGSVFENLRQQSADALAEIGFDGYAVGGLAVGEGQDEMFRVLDFSVPMLPDDKPHYLMGVGKPDDIVGAVERGIDMFDCVLPTRSGRNGQAFTWDGPINIRNARFSEDLKPLDSECHCAVCQKWSRAYIHHLIRAGEILGAMLMTEHNIAFYQQLMQKIRDSISEGRFSQFAQDFRARYFA                                               |CCCCEEEEEEEECCEEEEEEEECCEEEEECEEECECCCCCCCCCCHHHHHHCCCCCEEEEHHHHHHCCCHHHHHHCCCHHHHHCCCCCEEEECCHHHHCCECCCCEEEECCCEEEECHHHHHHHHHHHCCCEEECCCCCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHCCHHHHHHCEEEEEECCCCCHHHHHHHHHHHHHHCCCEEEECCCCCCCCHHHHHHHHHHHCCCCCCCCCEEECCECCHHHHHHHHHCCCCEEECCHHHHHHHCCEECCCCCCEECCCCCCCCCCCCCCCCCCCHHHHHCCHHHHHHHHHCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHCCCHHHHHHHHHHHHCC                                               |true   |\n",
      "|d4ac7b_|NYIVPGEYRVAEGEIEINAGREKTTIRVSNTGDRPIQVGSHIHFVEVNKELLFDRAEGIGRRLNIPSGTAARFEPGEEMEVELTELGGNREVFGISDLTNGSVDNKELILQRAKELGYKGVE                                                                                                                                                                                                                                                                                                 |CCCCCCCEECCCCEEECCCCCCEEEEEEEECCCCCEEEECCCCCCCCCCCEECCCCCCCCEEECCCCCCEEEECCCCEEEEEEEECCCCCEECCCCCCCCEECCCHHHHHHHHHHHCCCCCC                                                                                                                                                                                                                                                                                                 |true   |\n",
      "|d1vyra_|AEKLFTPLKVGAVTAPNRVFMAPLTRLRSIEPGDIPTPLMGEYYRQRASAGLIISEATQISAQAKGYAGAPGLHSPEQIAAWKKITAGVHAEDGRIAVQLWHTGRISHSSIQPGGQAPVSASALNANTRTSLRDENGNAIRVDTTTPRALELDEIPGIVNDFRQAVANAREAGFDLVELHSAHGYLLHQFLSPSSNQRTDQYGGSVENRARLVLEVVDAVCNEWSADRIGIRVSPIGTFQNVDNGPNEEADALYLIEELAKRGIAYLHMSETDLAGGKPYSEAFRQKVRERFHGVIIGAGAYTAEKAEDLIGKGLIDAVAFGRDYIANPDLVARLQKKAELNPQRPESFYGGGAEGYTDYPSL                                                |CCCCCCCEEECCEEECCCEEECCCCCCCCECCCCECCHHHHHHHHHCCCCCEEEEEEEECCCCCCCCCCCCECCCHHHHHHHHHHHHHHHHCCCCEEEEEECCCCCCCCCCCCCCCCCEECCCCCCCCEEEEECCCCCEEEEECCCCEECCCCCHHHHHHHHHHHHHHHHHCCCCEEEEEECCCCHHHHHHCCCCCCCCCCCCCCHHHHCHHHHHHHHHHHHHCCCCCEEEEECCCCCECCECCCCCHHHHHHHHHHHHHHCCCCEEEEECCECCECCCCCHHHHHHHHHHCCCEEEEECCCCHHHHHHHHHCCCCCEEEECHHHHHCCCHHHHHHHCCCCCCCCCCCCCCCCCCCCCCCCCC                                                |true   |\n",
      "|d2bbha1|PPGTLVYTGKYREDFEIEVMNYSIEEFREFKTTDVESVLPFRDSSTPTWINITGIHRTDVVQRVGEFFGTHPLVLEDILNVHQRPKVEFFENYVFIVLKMFTYDKHELESEQVSLILTKNCVLMFQEKIGDVFDPVRERIRYNRGIIRKKRADYLLYSLIDALVDDYFVLLEKIDDEIDVLEEEVTVQRTHQLKRNLVELRKTIWPLREVLSSLYRDVPPLIE                                                                                                                                                                                            |CCCCCCCCCCCCCCCEEEEEEEECCEEEEEEECCCCCCCCCCCCCCCEEEEEECCCCHHHHHHHHHHHCCCHHHHHHHHCCCCCCEEEECCCEEEEEEEEEECCCCCCEEEEEEEEEECCEEEEEECCCCCCCHHHHHHHHCCCCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCC                                                                                                                                                                                            |false  |\n",
      "|d1ozna_|PCPGACVCYNEPKVTTSCPQQGLQAVPVGIPAASQRIFLHGNRISHVPAASFRACRNLTILWLHSNVLARIDAAAFTGLALLEQLDLSDNAQLRSVDPATFHGLGRLHTLHLDRCGLQELGPGLFRGLAALQYLYLQDNALQALPDDTFRDLGNLTHLFLHGNRISSVPERAFRGLHSLDRLLLHQNRVAHVHPHAFRDLGRLMTLYLFANNLSALPTEALAPLRALQYLRLNDNPWVCDCRARPLWAWLQKFRGSSSEVPCSLPQRLAGRDLKRLAANDLQGC                                                                                                                               |CCCCCCEEECCCCCEEECCCCCCCCCCCCCCCCCCEEECCCCCCCEECCCCCCCCCCCCEEECCCCCCCEECCCCCCCCCCCCEEECCCCCCCCCCCCCCCCCCCCCCEEECCCCCCCCCCCCCCCCCCCCCEEECCCCCCCCCCCCCCCCCCCCCEEECCCCCCCEECCCCCCCCCCCCEEECCCCCCCEECCCCCCCCCCCCEEECCCCCCCCCCHHHHCCCCCCCEEECCCCCEECCCCCHHHHHHHHHCCCEECCCEEEECCCCCCCECCCCCCCCCCCC                                                                                                                               |true   |\n",
      "|d1gd8a_|SSHRLALYRNQAKSLLTHGRITTTVPKAKELRGFVDHLIHLAKRGDLHARRLVLRDLQDVKLVRKLFDEIAPRYRDRQGGYTRVLKLAERRRGDGAPLALVELVE                                                                                                                                                                                                                                                                                                                  |CHHHHHHHHHHHHHHHHHCEEEEEHHHHHHHHHHHHHHHHHHHHCCHHHHHHHHHHCCCHHHHHHHHHCHHHHCCCCCCCCEEEEEEEEECCCCCCEEEEEEECC                                                                                                                                                                                                                                                                                                                  |true   |\n",
      "|d3f6ya_|FWRQTWSGPGTTKRFPETVLARCVKYTEIHPEMRHVDCQSVWDAFKGAFISKHPCDITEEDYQPLMKLGTQTVPCNKILLWSRIKDLAHQFTQVQRDMFTLEDTLLGYLADDLTWCGEFDTSKINYQSCPDWRKDCSNNPVSVFWKTVSRRFAEAACDVVHVMLDGSRSKIFDKDSTFGSVEVHNLQPEKVQTLEAWVIHGGREDSRDLCQDPTIKELESIISKRNIQFSCKNIY                                                                                                                                                                                |CCCCCCCEECCCCCHHHHHHHHHHHHHHHCCCCCCCCHHHHHHHHHHHHCCCCCCCCCCCCCHHHHHHCCCCCCCCCEECCCCCCCCCCCCHHHHHHCECCCCCHHHHHHCCCCCCCCCCCCCCCCCEECCCCCCCCCCHHHHHHHHHHHHHHHHCCEEEEEEEECCCCCCCCCCCHHHHCCCCCCCCCCEEEEEEEEECCCCCCCCCCCCCHHHHHHHHHHHHCCCEEEEEEEC                                                                                                                                                                                |true   |\n",
      "|d2pjua1|KPVIWTVSVTRLFELFRDISLEFDHLANITPIQLGFEKAVTYIRKKLANERCDAIIAAGSNGAYLKSRLSVPVILIKPSGYDVLQFLAKAGKLTSSIGVVTYQETIPALVAFQKTFNLRLDQRSYITEEDARGQINELKANGTEAVVGAGLITDLAEEAGMTGIFIYSAATVRQAFSDALDMTRMS                                                                                                                                                                                                                                 |CCEEEEECCHHHHHHHHHHHCCCCCCCEEEEECCCHHHHHHHHHHHCCCCCCCEEEEEHHHHHHHHCCCCCCEEEECCCHHHHHHHHHHCCCCCCCEEEEEECCCCHHHHHHHHHHCCCEEEEEECCHHHHHHHHHHHHHCCCCEEEECHHHHHHHHHCCCEEEECCCHHHHHHHHHHHHHHHHHC                                                                                                                                                                                                                                 |true   |\n",
      "|d4g78a_|SNAMDHLHRKLRDHEAAMFQQGYLDDQFSQLQKLQDDTSPDFVIEVMTMFFDDSEKLLNNMSRALEQVPVNFKQIDAHAHQQKGSSASVGAARVKNVCGTFRNFCEAQNLEGCVRCLQQLQQEYSLLKNNLKYLFKLQQEIKTAGRS                                                                                                                                                                                                                                                                        |CCCCCHHHHHHHHHHHHHHHCCCECHHHHHHHCCCCCCCCHHHHHHHHHHHHHHHHHHHHHHHHHHCCCCCHHHHHHHHHHHHHHHHHHCEHHHHHHHHCHHHHHHCCCHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCC                                                                                                                                                                                                                                                                        |true   |\n",
      "+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "res.show(20, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coalesce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = res.coalesce(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Save to a local JSON file\n",
    "\n",
    "### This line of code will overwrite exsisting file or directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.write.mode(\"overwrite\").format(\"json\").save(\"Local directory to save your JSON file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Terminate Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}