methods.html

<!doctype html>
<!--
  Minimal Mistakes Jekyll Theme 4.16.4 by Michael Rose
  Copyright 2013-2019 Michael Rose - mademistakes.com | @mmistakes
  Free for personal and commercial use under the MIT license
  https://github.com/mmistakes/minimal-mistakes/blob/master/LICENSE
-->
<html lang="en" class="no-js">
  <head>
    <meta charset="utf-8">

<!-- begin _includes/seo.html --><title>Developing powerful, general-purpose interpretation methodologies - Interpreting Deep Learning</title>
<meta name="description" content="Website for 2019 NWA-ORC proposal BD.1910: ‘Interpreting Deep Learning Models for Text and Sound: Methods &amp; Applications’.">



<meta property="og:type" content="website">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="Interpreting Deep Learning">
<meta property="og:title" content="Developing powerful, general-purpose interpretation methodologies">
<meta property="og:url" content="http://localhost:4000/methods">




  <meta property="og:image" content="http://localhost:4000/assets/images/network-bw-1.png">









  

  


<link rel="canonical" href="http://localhost:4000/methods">







  <script type="application/ld+json">
    {
      "@context": "https://schema.org",
      "@type": "Person",
      "name": "InterpretingDL",
      "url": "http://localhost:4000",
      "sameAs": null
    }
  </script>







<!-- end _includes/seo.html -->


<link href="/feed.xml" type="application/atom+xml" rel="alternate" title="Interpreting Deep Learning Feed">

<!-- https://t.co/dKP3o1e -->
<meta name="viewport" content="width=device-width, initial-scale=1.0">

<script>
  document.documentElement.className = document.documentElement.className.replace(/\bno-js\b/g, '') + ' js ';
</script>

<!-- For all browsers -->
<link rel="stylesheet" href="/assets/css/main.css">

<!--[if IE ]>
  <style>
    /* old IE unsupported flexbox fixes */
    .greedy-nav .site-title {
      padding-right: 3em;
    }
    .greedy-nav button {
      position: absolute;
      top: 0;
      right: 0;
      height: 100%;
    }
  </style>
<![endif]-->



    <!-- start custom head snippets -->

<!-- insert favicons. use https://realfavicongenerator.net/ -->
<link rel="apple-touch-icon" sizes="180x180" href="/assets/images/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/assets/images/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/assets/images/favicon-16x16.png">
<link rel="manifest" href="/assets/images/site.webmanifest">
<link rel="mask-icon" href="/assets/images/safari-pinned-tab.svg" color="#5bbad5">
<link rel="shortcut icon" href="/assets/images/favicon.ico">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/assets/images/browserconfig.xml">
<meta name="theme-color" content="#ffffff">

<!-- end custom head snippets -->

  </head>

  <body class="layout--single wide">

    <!--[if lt IE 9]>
<div class="notice--danger align-center" style="margin: 0;">You are using an <strong>outdated</strong> browser. Please <a href="https://browsehappy.com/">upgrade your browser</a> to improve your experience.</div>
<![endif]-->

    

<div class="masthead">
  <div class="masthead__inner-wrap">
    <div class="masthead__menu">
      <nav id="site-nav" class="greedy-nav">
        
          <a class="site-logo" href="/"><img src="/assets/images/brain.png" alt=""></a>
        
        <a class="site-title" href="/">InterpretingDL</a>
        <ul class="visible-links"></ul>
        
        <button class="greedy-nav__toggle hidden" type="button">
          <span class="visually-hidden">Toggle menu</span>
          <div class="navicon"></div>
        </button>
        <ul class="hidden-links hidden"></ul>
      </nav>
    </div>
  </div>
</div>


    <div class="initial-content">
      
  











<div class="page__hero--overlay"
  style="background-color: #5e616c; background-image: url('/assets/images/network-bw-1.png');"
>
  
    <div class="wrapper">
      <h1 id="page-title" class="page__title" itemprop="headline">
        
          Developing powerful, general-purpose interpretation methodologies

        
      </h1>
      
      
      
      
    </div>
  
  
</div>





<div id="main" role="main">
  
  <div class="sidebar sticky">
  
  
    
      
      
      
      
    
    
      

<nav class="nav__list">
  
  <input id="ac-toc" name="accordion-toc" type="checkbox" />
  <label for="ac-toc">Toggle Menu</label>
  <ul class="nav__items">

      <li>
          <a href="/"><span class="nav__sub-title">Home</span></a>
      </li>

      <li>
          <a href="/projects"><span class="nav__sub-title">Projects</span></a>
      </li>


      <li>
          <a href="/methods"><span class="nav__sub-title">Methods</span></a>
      </li>

      <li>
          <a href="/papers"><span class="nav__sub-title">Key papers</span></a>
      </li>

    
  </ul>
</nav>
    
  
  </div>


  <article class="page" itemscope itemtype="https://schema.org/CreativeWork">
    <meta itemprop="headline" content="Developing powerful, general-purpose interpretation methodologies">
    
    
    

    <div class="page__inner-wrap">
      

      <section class="page__content" itemprop="text">
        
        <p>One of the main goals of the InterpretingDL network is to develop a general-purpose framework for applying appropriate analysis and interpretation methods to neural network models of text and speech. We focus on two types of interpretation techniques to evaluate neural models: <em>data-driven</em> and <em>hypothesis-driven</em> methods.</p>

<p>Hypothesis-driven methods test whether specific a priori defined information can be decoded from the internal states of a neural model. For instance, hypothesis-driven methods can be used to answer questions like “Does my translation model encode morphology?”.</p>

<p>Data-driven methods instead do not require a specific hypothesis about what information is encoded, but can consider in a more open way how representations are formed. Data-driven methods are more suitable for open questions such as “What kind of words are the main drivers for a particular translation”.</p>

<p>We aim to <em>advance</em> and <em>integrate</em> both types of interpretation techniques, as we explain below.</p>

<h1 id="hypothesis-driven-methods">Hypothesis-driven methods</h1>
<p>A variety of analysis techniques have been proposed in the academic literature for better understanding the representations learned by deep learning models of language. The main families of such methods include <em>diagnostic classifiers</em> (or <em>probing techniques</em>), <em>representational similarity analysis</em> and <em>canonical correlation analysis</em>.</p>

<p><a name="dc"></a> <strong>Diagnostic classifiers</strong> often use the activations from different layers of a deep learning architecture as input to a prediction model. Success in predicting the target information is taken as evidence that the original model encodes that information in order to perform the task it is optimized for.</p>

<p><a name="rsa"></a> <strong>Representational Similarity Analysis (RSA)</strong> methods are borrowed from neuroscience, and used for correlating data points from two different representation spaces. When applied to deep learning models, layer activations construct one such representation space and are compared to the (structured) linguistic representations of each data item.</p>

<p><a name="cca"></a> <strong>Canonical correlation analysis (CCA)</strong> techniques allow activations of a deep NLP model to be correlated with those of a parallel predictive model through estimating the strength of association between two canonical variates.</p>

<p>Each of these categories of techniques offer strengths and weaknesses. Our focus is to understand the strengths and weaknesses of each category of techniques, and identify which type of network architecture and analysis task they are best suitable for. Furthermore, we will develop integrated methods that combine the exploratory power of each type of technique<sup id="fnref:1"><a href="#fn:1" class="footnote">1</a></sup>. Finally, we will develop a unified framework for applying different interpretation techniques to each case, depending on the characteristics of the task, the modality of the input and the architecture of the model under investigation.</p>

<h1 id="data-driven-methods">Data-driven methods</h1>
<p>Next to evaluating the representations learned by neural networks, we are also concerned with <em>how</em> and <em>why</em> this information ended up in those representations. In particular, we aim to integrate two techniques that can pin-point which events are responsible for the representation of a model at any particular point in time: <em>layer-wise relevance propagation</em> and <em>contextual decomposition</em>.</p>

<p><a name="lrp"></a> <strong>Layer-wise Relevance Propagation (LRP)</strong> is a method that has worked very well in the domain of vision. It has recently been adapted for the types of models that are typically used for processing text and speech. LRP defines specific gradient-based rules that quantify how ‘relevance’ propagates back through the network. For any particular state of the model, this allows to compute a relevance score for any input that the model received before.</p>

<p><a name="cd"></a> <strong>Contextual Decomposition (CD)</strong> is a forward method, that uses clever linearisations of the non-linear network dynamics to compute how the information stemming from a particular input or model component propagates forward to the model and contributes to the decisions the model makes later on. Importantly, the generalised version of this method (GCD) also allows to understand how decisions depend on the biases of the model itself, irrespective of the information it receives (see <a href="#fig1">Figure 1</a>).</p>

<p><a name="fig1"></a><figure class="image">
  <img src="../assets/images/rnn_diagram_hc.png" alt="**Figure 1.** Diagram of an LSTM (unfolded in time), that takes inputs x and produces output y, both changing over time. Shown are the partitionings of the 'memory cell' **c** and the hidden layer **h** as sums of 15 and 9 terms each, as used in the _Contextual Decomposition_ technique (Murdoch et al., 2018; Jumelet et al., 2019)." />
</figure></p>
<figcaption>
  <p><strong>Figure 1.</strong> Diagram of an LSTM (unfolded in time), that takes inputs x and produces output y, both changing over time. Shown are the partitionings of the ‘memory cell’ <strong>c</strong> and the hidden layer <strong>h</strong> as sums of 15 and 9 terms each, as used in the <em>Contextual Decomposition</em> technique (Murdoch et al., 2018; Jumelet et al., 2019).</p>
</figcaption>

<p>Both data-driven methods again have their own advantages and disadvantages. We aim to combine the advantages of both methods to create a <em>best-of-both worlds</em> technique that can show in a computationally efficient way which inputs and model components are responsible for the current model’s decision. We will define and test this technique for several different types of models with different non-linear dynamics and compare the results. Furthermore, we add a specific focus to the influence of the model’s biases, a better understanding of which may play an important role in the development of models that make more fair and predictable decisions.</p>

<div class="footnotes">
  <ol>
    <li id="fn:1">
      <p>A study combining diagnostic classifiers and RSA-based techniques has been recently published by <a href="https://arxiv.org/abs/1905.06401">Chrupała &amp; Alishahi at ACL 2019</a>. <a href="#fnref:1" class="reversefootnote">&#8617;</a></p>
    </li>
  </ol>
</div>

        
      </section>

      <footer class="page__meta">
        
        


        
      </footer>

      

      
    </div>

    
  </article>

  
  
</div>

    </div>

    

    <div class="page__footer">
      <footer>
        <!-- start custom footer snippets -->

<!-- end custom footer snippets -->

        <div class="page__footer-follow">
  <ul class="social-icons">
    

    

    <li><a href="/feed.xml"><i class="fas fa-fw fa-rss-square" aria-hidden="true"></i> Feed</a></li>
  </ul>
</div>

<div class="page__footer-copyright">&copy; 2020 InterpretingDL. Powered by <a href="https://jekyllrb.com" rel="nofollow">Jekyll</a> &amp; <a href="https://mademistakes.com/work/minimal-mistakes-jekyll-theme/" rel="nofollow">Minimal Mistakes</a>.</div>

      </footer>
    </div>

    
  <script src="/assets/js/main.min.js"></script>
  <script defer src="https://use.fontawesome.com/releases/v5.8.2/js/all.js" integrity="sha384-DJ25uNYET2XCl5ZF++U8eNxPWqcKohUUBUpKGlNLMchM7q4Wjg2CUpjHLaL8yYPH" crossorigin="anonymous"></script>










  </body>
</html>