RL-with-world-model.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->

<svg
   width="210mm"
   height="600mm"
   viewBox="0 0 210 600"
   version="1.1"
   id="svg1268"
   inkscape:version="1.2.2 (1:1.2.2+202212051550+b0a8486541)"
   sodipodi:docname="RL-with-world-model.svg"
   xml:space="preserve"
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns:xlink="http://www.w3.org/1999/xlink"
   xmlns="http://www.w3.org/2000/svg"
   xmlns:svg="http://www.w3.org/2000/svg"><sodipodi:namedview
     id="namedview1270"
     pagecolor="#ffffff"
     bordercolor="#cccccc"
     borderopacity="1"
     inkscape:showpageshadow="0"
     inkscape:pageopacity="1"
     inkscape:pagecheckerboard="0"
     inkscape:deskcolor="#d1d1d1"
     inkscape:document-units="mm"
     showgrid="false"
     inkscape:zoom="1.4142136"
     inkscape:cx="405.17219"
     inkscape:cy="1336.7854"
     inkscape:window-width="1080"
     inkscape:window-height="1895"
     inkscape:window-x="0"
     inkscape:window-y="0"
     inkscape:window-maximized="1"
     inkscape:current-layer="layer1"
     inkscape:pageshadow="2" /><defs
     id="defs1265"><filter
       style="color-interpolation-filters:sRGB"
       inkscape:label="Colorize"
       id="filter68641"
       x="0"
       y="0"
       width="1"
       height="1"><feColorMatrix
         values="0.21 0.72 0.072 0 0 0.21 0.72 0.072 0 0 0.21 0.72 0.072 0 0 0 0 0 1 0 "
         id="feColorMatrix68639"
         result="fbSourceGraphic" /><feColorMatrix
         result="fbSourceGraphicAlpha"
         in="fbSourceGraphic"
         values="0 0 0 -1 0 0 0 0 -1 0 0 0 0 -1 0 0 0 0 1 0"
         id="feColorMatrix1833" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1835"
         operator="arithmetic"
         k1="0.5"
         k2="0.5"
         result="composite1"
         in="fbSourceGraphic"
         k3="0"
         k4="0" /><feColorMatrix
         id="feColorMatrix1837"
         in="composite1"
         values="1"
         type="saturate"
         result="colormatrix1" /><feFlood
         id="feFlood1839"
         flood-opacity="0.372549"
         flood-color="rgb(255,253,0)"
         result="flood1" /><feBlend
         in2="colormatrix1"
         id="feBlend1841"
         in="flood1"
         mode="multiply"
         result="blend1" /><feBlend
         in2="blend1"
         id="feBlend1843"
         mode="normal"
         result="blend2" /><feColorMatrix
         id="feColorMatrix1845"
         in="blend2"
         values="1"
         type="saturate"
         result="colormatrix2" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1847"
         in="colormatrix2"
         operator="in"
         result="fbSourceGraphic" /><feColorMatrix
         result="fbSourceGraphicAlpha"
         in="fbSourceGraphic"
         values="0 0 0 -1 0 0 0 0 -1 0 0 0 0 -1 0 0 0 0 1 0"
         id="feColorMatrix1849" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1851"
         operator="arithmetic"
         k1="0.5"
         k2="0.5"
         result="composite1"
         in="fbSourceGraphic"
         k3="0"
         k4="0" /><feColorMatrix
         id="feColorMatrix1853"
         in="composite1"
         values="1"
         type="saturate"
         result="colormatrix1" /><feFlood
         id="feFlood1855"
         flood-opacity="0.372549"
         flood-color="rgb(255,218,0)"
         result="flood1" /><feBlend
         in2="colormatrix1"
         id="feBlend1857"
         in="flood1"
         mode="multiply"
         result="blend1" /><feBlend
         in2="blend1"
         id="feBlend1859"
         mode="normal"
         result="blend2" /><feColorMatrix
         id="feColorMatrix1861"
         in="blend2"
         values="1"
         type="saturate"
         result="colormatrix2" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1863"
         in="colormatrix2"
         operator="in"
         result="fbSourceGraphic" /><feColorMatrix
         result="fbSourceGraphicAlpha"
         in="fbSourceGraphic"
         values="0 0 0 -1 0 0 0 0 -1 0 0 0 0 -1 0 0 0 0 1 0"
         id="feColorMatrix1977" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1979"
         operator="arithmetic"
         k1="0.7"
         k2="0.7"
         result="composite1"
         in="fbSourceGraphic"
         k3="0"
         k4="0" /><feColorMatrix
         id="feColorMatrix1981"
         in="composite1"
         values="1"
         type="saturate"
         result="colormatrix1" /><feFlood
         id="feFlood1983"
         flood-opacity="1"
         flood-color="rgb(255,252,0)"
         result="flood1" /><feBlend
         in2="colormatrix1"
         id="feBlend1985"
         in="flood1"
         mode="multiply"
         result="blend1" /><feBlend
         in2="blend1"
         id="feBlend1987"
         mode="normal"
         result="blend2" /><feColorMatrix
         id="feColorMatrix1989"
         in="blend2"
         values="1"
         type="saturate"
         result="colormatrix2" /><feComposite
         in2="fbSourceGraphic"
         id="feComposite1991"
         in="colormatrix2"
         operator="in"
         result="composite2" /></filter><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="5.3244081"
       markerHeight="6.155385"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="strokeWidth"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-3"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="4"
       markerHeight="4.6242776"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="userSpaceOnUse"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-6" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-1"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="5.3244081"
       markerHeight="6.155385"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="strokeWidth"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-2" /></marker><marker
       style="overflow:visible"
       id="Arrow2Send"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Send"><path
         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3909"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow1Lend"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow1Lend"><path
         transform="matrix(-0.8,0,0,-0.8,-10,0)"
         style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"
         d="M 0,0 5,-5 -12.5,0 5,5 0,0 z"
         id="path3879"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Mend"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Mend"><path
         transform="scale(-0.6,-0.6)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3903"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow1Mend"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow1Mend"><path
         transform="matrix(-0.4,0,0,-0.4,-4,0)"
         style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"
         d="M 0,0 5,-5 -12.5,0 5,5 0,0 z"
         id="path3885"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-2"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-9"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-1"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-94"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-8"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-4"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-0"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-3"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-10"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-6"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-06"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-1"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2Lend-4"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2Lend"><path
         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
         id="path3897-7"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2MendS"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2MendS"><path
         transform="scale(-0.6,-0.6)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill:#0000ff;fill-rule:evenodd;stroke:#0000ff;stroke-width:0.625;stroke-linejoin:round"
         id="path7051"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2MendSG"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2MendSG"><path
         transform="scale(-0.6,-0.6)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill:#ff0000;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round"
         id="path7329"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="Arrow2MendSGT"
       refX="0"
       refY="0"
       orient="auto"
       inkscape:stockid="Arrow2MendSGT"><path
         transform="scale(-0.6,-0.6)"
         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
         style="fill:#000000;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round"
         id="path4633"
         inkscape:connector-curvature="0" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-36"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="5.3244081"
       markerHeight="6.155385"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="strokeWidth"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-7" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-36-2"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="5.3244081"
       markerHeight="6.155385"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="strokeWidth"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-7-0" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-3-9"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="4"
       markerHeight="4.6242776"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="userSpaceOnUse"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-6-3" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-3-94"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="4"
       markerHeight="4.6242776"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="userSpaceOnUse"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-6-7" /></marker><marker
       style="overflow:visible"
       id="TriangleStart-7-5-6-36-2-7"
       refX="0"
       refY="0"
       orient="auto-start-reverse"
       inkscape:stockid="TriangleStart"
       markerWidth="5.3244081"
       markerHeight="6.155385"
       viewBox="0 0 5.3244081 6.1553851"
       inkscape:isstock="true"
       inkscape:collect="always"
       preserveAspectRatio="xMidYMid"
       markerUnits="strokeWidth"><path
         transform="scale(0.5)"
         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
         d="M 5.77,0 -2.88,5 V -5 Z"
         id="path135-5-6-1-7-0-4" /></marker></defs><g
     inkscape:label="main"
     inkscape:groupmode="layer"
     id="layer1"><path
       id="rect390-3-5"
       style="fill:none;stroke:#000000;stroke-width:0.99219;stop-color:#000000"
       d="m 143.39611,26.31879 c 0,-7.08482 5.70365,-12.78847 12.78845,-12.78847 l 12.30063,0.008 c 7.0848,0 12.78845,5.70365 12.78845,12.78846 v 15.875 c 0,7.0848 -5.70365,12.788447 -12.78845,12.788447 l -12.30063,-0.008 c -7.0848,0 -12.78845,-5.703637 -12.78845,-12.788437 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="RL-with-autoencoder.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><rect
       style="fill:#ffffff;stroke-width:1.05833;stroke-linecap:round;stroke-dasharray:none;stop-color:#000000"
       id="rect1432"
       width="5.5612483"
       height="6.0122695"
       x="140.80843"
       y="20.501799"
       ry="0" /><g
       id="g1378"
       transform="translate(58.481928,-126.56082)"><path
         id="rect67505-5"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 68.242985,153.773 16.198577,3.81001 v 10.47214 l -16.198577,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="rect67505-8-3"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 102.1581,153.773 -16.198598,3.81001 v 10.47214 l 16.198598,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="use69167-4-1"
         style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 83.513866,157.51966 h 3.244584 v 10.47216 h -3.244584 z"
         sodipodi:nodetypes="ccccc"
         inkscape:tile-x0="27.512444"
         inkscape:tile-y0="-320.67129" /></g><path
       style="fill:none;stroke:#000000;stroke-width:0.85926;marker-end:url(#TriangleStart-7-5-6);stop-color:#000000"
       d="m 162.11657,13.53805 h 1.01888"
       id="path65933-9-1"
       sodipodi:nodetypes="cc" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:129.108;display:inline;stroke-width:0.264583"
       x="21.446598"
       y="64.498283"
       id="text4811"
       transform="translate(-15.355531,-6.9032932)" /><g
       id="g7102-9"
       transform="translate(132.65314,-428.10423)"><rect
         style="fill:#cccccc;stroke-width:0.264583;stroke-dasharray:0.529167, 0.529167;-inkscape-stroke:none;stop-color:#000000"
         id="rect330-2-2-2-0-4"
         width="15.49725"
         height="16.300793"
         x="40.667614"
         y="453.5441"
         ry="7.2534728" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;fill:#ff0000;stroke-width:0.264583"
         x="44.599758"
         y="463.78436"
         id="text233-3-9-2-3-8"><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';fill:#ff0000;stroke-width:0.264583"
           x="44.599758"
           y="463.78436"
           id="tspan274-2-8-6-1">RL</tspan></text></g><path
       id="rect390-3-5-7"
       style="fill:none;stroke:#000000;stroke-width:0.99219;stop-color:#000000"
       d="m 42.930853,27.821601 c 0,-7.08482 5.703647,-12.788463 12.788431,-12.788463 l 12.300639,0.008 c 7.0848,0 12.78845,5.703643 12.78845,12.788453 v 5.29166 c 0,7.08479 -5.70365,12.78844 -12.78845,12.78844 l -12.300639,-0.008 c -7.084784,0 -12.788431,-5.70364 -12.788431,-12.78843 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="minimal-RL.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><path
       style="fill:none;stroke:#000000;stroke-width:0.85926;marker-end:url(#TriangleStart-7-5-6-3);stop-color:#000000"
       d="m 61.651305,15.040871 h 1.01888"
       id="path65933-9-1-5"
       sodipodi:nodetypes="cc" /><g
       id="g7102-9-3"
       transform="translate(32.187879,-431.8931)"><rect
         style="fill:#cccccc;stroke-width:0.264583;stroke-dasharray:0.529167, 0.529167;-inkscape-stroke:none;stop-color:#000000"
         id="rect330-2-2-2-0-4-5"
         width="15.49725"
         height="16.300793"
         x="40.667614"
         y="453.5441"
         ry="7.2534728" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;fill:#ff0000;stroke-width:0.264583"
         x="44.599758"
         y="463.78436"
         id="text233-3-9-2-3-8-6"><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';fill:#ff0000;stroke-width:0.264583"
           x="44.599758"
           y="463.78436"
           id="tspan274-2-8-6-1-2">RL</tspan></text></g><g
       id="g1235"
       transform="translate(-44.937486,-59.234304)"><path
         id="use69167-4-1-9"
         style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.05833;stroke-dasharray:none;stop-color:#000000"
         d="m 77.434593,93.441664 v -8.656001 h 21.033263 v 8.656001 z"
         sodipodi:nodetypes="ccccc"
         inkscape:tile-x0="27.512444"
         inkscape:tile-y0="-320.67129" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
         x="80.567101"
         y="91.155724"
         id="text1181"><tspan
           sodipodi:role="line"
           id="tspan1179"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi';stroke-width:0.264583"
           x="80.567101"
           y="91.155724">state</tspan></text></g><text
       xml:space="preserve"
       style="font-weight:300;font-size:5.08px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;fill:#ff0000;stroke-width:0.211666"
       x="126.8994"
       y="24.813139"
       id="text233-31-6"><tspan
         sodipodi:role="line"
         id="tspan231-4-8"
         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';fill:#ff0000;stroke-width:0.211666"
         x="126.8994"
         y="24.813139">auto-encoder</tspan></text><path
       id="rect390-3-5-70"
       style="fill:none;stroke:#000000;stroke-width:0.99219;stop-color:#000000"
       d="m 35.398542,84.247119 c 0,-7.084805 5.70365,-9.084286 12.788451,-9.084286 l 8.067291,0.008 c 7.084797,0 12.788441,1.999481 12.788441,9.084276 v 2.11666 c 0,7.084797 -5.703644,9.084284 -12.788441,9.084284 l -8.067291,-0.008 c -7.084801,0 -12.788451,-1.999477 -12.788451,-9.084274 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="RL-with-autoencoder-TTT.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><path
       id="rect67505-5-6"
       style="opacity:1;fill:#cccccc;fill-opacity:0.8;stroke:#000000;stroke-width:1.13944;stroke-dasharray:none;stop-color:#000000"
       d="m 25.792205,99.867953 4.222346,-14.525814 h 11.60548 l 4.222335,14.525814 z"
       sodipodi:nodetypes="ccccc" /><path
       id="rect67505-8-3-0"
       style="opacity:1;fill:#cccccc;fill-opacity:0.8;stroke:#000000;stroke-width:1.13944;stroke-dasharray:none;stop-color:#000000"
       d="m 25.792205,69.455132 4.222346,14.52582 h 11.60548 l 4.222335,-14.52582 z"
       sodipodi:nodetypes="ccccc" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:5.08px;line-height:1.05;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;fill:#ff0000;stroke-width:0.211666"
       x="17.347649"
       y="83.381653"
       id="text233-31-6-9"><tspan
         sodipodi:role="line"
         id="tspan231-4-8-2"
         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.211666"
         x="17.347649"
         y="83.381653">auto-</tspan><tspan
         sodipodi:role="line"
         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.211666"
         x="17.347649"
         y="88.715652"
         id="tspan13381">encoder</tspan></text><g
       id="g10632"
       transform="matrix(0.8,0,0,0.8,14.298527,-79.999644)"><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 19.711798,232.35604 H 33.31515"
         id="path2125"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 19.711798,237.64771 H 33.31515"
         id="path2125-3"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 24.66139,241.80354 V 228.20018"
         id="path2125-3-6"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 28.894723,241.80354 V 228.20018"
         id="path2125-3-6-7"
         sodipodi:nodetypes="cc" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:4.93889px;line-height:1.05;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;stroke-width:0.264583"
         x="20.933023"
         y="231.27473"
         id="text709"><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="231.27473"
           id="tspan8113">x</tspan><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="236.46057"
           id="tspan8117">o o </tspan><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="241.64641"
           id="tspan9299">x</tspan></text></g><g
       id="g10632-5"
       transform="matrix(0.8,0,0,0.8,14.298527,-127.09552)"><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 19.711798,232.35604 H 33.31515"
         id="path2125-35"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 19.711798,237.64771 H 33.31515"
         id="path2125-3-62"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 24.66139,241.80354 V 228.20018"
         id="path2125-3-6-9"
         sodipodi:nodetypes="cc" /><path
         style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
         d="M 28.894723,241.80354 V 228.20018"
         id="path2125-3-6-7-1"
         sodipodi:nodetypes="cc" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:4.93889px;line-height:1.05;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;stroke-width:0.264583"
         x="20.933023"
         y="231.27473"
         id="text709-2"><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="231.27473"
           id="tspan8113-7">x</tspan><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="236.46057"
           id="tspan8117-0">o o x</tspan><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans';text-align:start;text-anchor:start;stroke-width:0.264583"
           x="20.933023"
           y="241.64641"
           id="tspan9299-9">x</tspan></text></g><path
       style="fill:none;stroke:#000000;stroke-width:0.85926;marker-end:url(#TriangleStart-7-5-6-1);stop-color:#000000"
       d="m 52.002332,75.170573 h 1.01888"
       id="path65933-9-1-2"
       sodipodi:nodetypes="cc" /><g
       id="g7102-9-6"
       transform="translate(19.158518,-376.90559)"><rect
         style="fill:#cccccc;stroke-width:0.264583;stroke-dasharray:0.529167, 0.529167;-inkscape-stroke:none;stop-color:#000000"
         id="rect330-2-2-2-0-4-1"
         width="15.49725"
         height="16.300793"
         x="40.667614"
         y="453.5441"
         ry="7.2534728" /><text
         xml:space="preserve"
         style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;fill:#ff0000;stroke-width:0.264583"
         x="44.599758"
         y="463.78436"
         id="text233-3-9-2-3-8-8"><tspan
           sodipodi:role="line"
           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Bold';fill:#ff0000;stroke-width:0.264583"
           x="44.599758"
           y="463.78436"
           id="tspan274-2-8-6-1-7">RL</tspan></text></g><path
       id="use69167-4-1-6"
       style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.13944;stroke-dasharray:none;stop-color:#000000"
       d="m 29.944345,86.174039 v -2.909537 h 11.605502 v 2.909537 z"
       sodipodi:nodetypes="ccccc"
       inkscape:tile-x0="27.512444"
       inkscape:tile-y0="-320.67129" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.93889px;line-height:1.25;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;stroke-width:0.264583"
       x="55.37542"
       y="62.235729"
       id="text15563"><tspan
         sodipodi:role="line"
         id="tspan15561"
         style="font-size:4.93889px;stroke-width:0.264583"
         x="55.37542"
         y="62.235729">next-state</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;white-space:pre;inline-size:165.164;display:inline;stroke-width:0.264583"
       x="73.354286"
       y="197.01485"
       id="text17010"
       transform="translate(47.582012,-52.400604)" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
       x="8.3995275"
       y="279.38193"
       id="text593"><tspan
         sodipodi:role="line"
         style="stroke-width:0.264583"
         x="8.3995275"
         y="279.38193"
         id="tspan735" /></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
       x="8.6796923"
       y="328.9068"
       id="text593-5"><tspan
         sodipodi:role="line"
         style="stroke-width:0.264583"
         x="8.6796923"
         y="328.9068"
         id="tspan7796" /></text><rect
       style="fill:#ffff00;stroke:#000000;stroke-width:0.529167;stroke-linecap:round;stroke-dasharray:none;stop-color:#000000"
       id="rect2360"
       width="15"
       height="14"
       x="42.737225"
       y="129.17976"
       inkscape:tile-cx="50.456706"
       inkscape:tile-cy="317.03667"
       inkscape:tile-w="14.529167"
       inkscape:tile-h="14.529167"
       inkscape:tile-x0="43.192123"
       inkscape:tile-y0="309.77209"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(14.529167)"
       id="use2448"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(29.058333)"
       id="use2450"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(43.5875)"
       id="use2452"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(58.116667)"
       id="use2454"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(72.645833)"
       id="use2456"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(87.175)"
       id="use2458"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><rect
       style="fill:#ffff00;stroke:#000000;stroke-width:0.423334;stroke-linecap:round;stroke-dasharray:none;stop-color:#000000"
       id="rect2360-3"
       width="12"
       height="11.2"
       x="15.221666"
       y="172.46458"
       inkscape:tile-cx="50.456706"
       inkscape:tile-cy="317.03667"
       inkscape:tile-w="14.529167"
       inkscape:tile-h="14.529167"
       inkscape:tile-x0="43.192123"
       inkscape:tile-y0="309.77209"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(11.623334)"
       id="use2448-6"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(23.246666)"
       id="use2450-7"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(34.87)"
       id="use2452-5"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(46.493332)"
       id="use2454-3"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(58.116668)"
       id="use2456-5"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3"
       xlink:href="#rect2360-3"
       transform="translate(69.739996)"
       id="use2458-6"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360"
       xlink:href="#rect2360"
       transform="translate(117.57918)"
       id="use2460"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
       x="164.47124"
       y="138.81575"
       id="text2490"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan2488"
         style="stroke-width:0.264583"
         x="164.47124"
         y="138.81575">δx </tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
       x="151.79886"
       y="138.40251"
       id="text2490-2"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-delta.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan2488-9"
         style="stroke-width:0.264583"
         x="151.79886"
         y="138.40251">+</tspan></text><path
       sodipodi:nodetypes="csscssccsscsscc"
       inkscape:connector-curvature="0"
       id="path7003"
       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:300px;line-height:125%;font-family:'TeX Gyre Pagella';-inkscape-font-specification:'TeX Gyre Pagella';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#4d4d4d;fill-opacity:1;stroke:none;stroke-width:0.123001"
       d="M 93.134061,170.03454 C 91.7355,169.16983 88.93413,168.60899 82.439182,168.39999 l -14.232423,-0.45798 c -7.974816,-0.25662 -10.201271,-1.14276 -11.9289,-2.1067 -1.645361,0.93559 -3.952962,1.89015 -11.188497,2.1067 l -15.301905,0.45798 c -6.495526,0.19441 -9.049519,0.74149 -10.777153,1.63455 0.08226,-0.86471 0.658149,-0.832 2.138977,-1.15803 1.892172,-0.39692 5.021206,-0.88907 10.036739,-1.03918 l 15.301918,-0.45798 c 5.919961,-0.17718 8.14456,-1.61492 9.789921,-2.45127 1.645366,0.87888 3.459823,2.23334 10.03673,2.45127 l 13.821079,0.45798 c 5.672577,0.18797 8.884972,0.62809 10.777142,0.99665 1.563102,0.31186 2.138988,0.27915 2.221251,1.20056"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:7.05556px;line-height:1.25;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;stroke-width:0.264583"
       x="56.130962"
       y="161.80775"
       id="text9891"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan9889"
         style="stroke-width:0.264583"
         x="56.130962"
         y="161.80775">9 TicTacToe states</tspan></text><rect
       style="fill:#ff7f2a;stroke:#000000;stroke-width:0.423334;stroke-linecap:round;stroke-dasharray:none;stop-color:#000000"
       id="rect2360-3-2"
       width="12"
       height="11.2"
       x="102.53415"
       y="172.46458"
       inkscape:tile-cx="50.456706"
       inkscape:tile-cy="317.03667"
       inkscape:tile-w="14.529167"
       inkscape:tile-h="14.529167"
       inkscape:tile-x0="43.192123"
       inkscape:tile-y0="309.77209"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(11.623334)"
       id="use2448-6-6"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(23.246666)"
       id="use2450-7-1"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(34.87)"
       id="use2452-5-8"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(46.493332)"
       id="use2454-3-7"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(58.116668)"
       id="use2456-5-9"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#rect2360-3-2"
       xlink:href="#rect2360-3-2"
       transform="translate(69.739996)"
       id="use2458-6-2"
       width="100%"
       height="100%"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><path
       sodipodi:nodetypes="csscssccsscsscc"
       inkscape:connector-curvature="0"
       id="path7003-0"
       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:300px;line-height:125%;font-family:'TeX Gyre Pagella';-inkscape-font-specification:'TeX Gyre Pagella';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#4d4d4d;fill-opacity:1;stroke:none;stroke-width:0.123001"
       d="m 180.44659,170.03454 c -1.39856,-0.86471 -4.19993,-1.42555 -10.69488,-1.63455 l -14.23243,-0.45798 c -7.97482,-0.25662 -10.20128,-1.14276 -11.92891,-2.1067 -1.64536,0.93559 -3.95296,1.89015 -11.1885,2.1067 l -15.30192,0.45798 c -6.49552,0.19441 -9.04951,0.74149 -10.77714,1.63455 0.0823,-0.86471 0.65815,-0.832 2.13898,-1.15803 1.89216,-0.39692 5.02119,-0.88907 10.03673,-1.03918 l 15.30192,-0.45798 c 5.91997,-0.17718 8.14457,-1.61492 9.78993,-2.45127 1.64537,0.87888 3.45983,2.23334 10.03674,2.45127 l 13.82109,0.45798 c 5.67258,0.18797 8.88497,0.62809 10.77714,0.99665 1.56311,0.31186 2.13899,0.27915 2.22125,1.20056"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:7.05556px;line-height:1.25;font-family:'Alibaba Sans';-inkscape-font-specification:'Alibaba Sans, Light';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;stroke-width:0.264583"
       x="143.4435"
       y="161.80775"
       id="text9891-2"
       inkscape:export-filename="/home/yky/misc-programs/reinforcement-learning-experiments/state-with-intermediates.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan9889-3"
         style="stroke-width:0.264583"
         x="143.4435"
         y="161.80775">N intermediate states</tspan></text><path
       id="rect390-3-5-5"
       style="fill:none;stroke:#000000;stroke-width:0.99219;stop-color:#000000"
       d="m 40.679157,296.3414 c 0,-7.08482 5.703651,-12.78847 12.788451,-12.78847 l 12.300625,0.008 c 7.084804,0 12.788451,5.70365 12.788451,12.78846 v 14.81667 c 0,7.0848 -5.703647,12.78845 -12.788451,12.78845 l -12.300625,-0.008 c -7.0848,0 -12.788451,-5.70364 -12.788451,-12.78844 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="AE-architecture-2.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><g
       id="g1378-5"
       transform="translate(-44.235024,142.40346)"><path
         id="rect67505-5-62"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 68.242985,153.773 16.198577,3.81001 v 10.47214 l -16.198577,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="rect67505-8-3-9"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 102.1581,153.773 -16.198598,3.81001 v 10.47214 l 16.198598,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="use69167-4-1-1"
         style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 83.513866,157.51966 h 3.244584 v 10.47216 h -3.244584 z"
         sodipodi:nodetypes="ccccc"
         inkscape:tile-x0="27.512444"
         inkscape:tile-y0="-320.67129" /></g><path
       style="fill:none;stroke:#000000;stroke-width:0.85926;marker-end:url(#TriangleStart-7-5-6-36);stop-color:#000000"
       d="m 62.964918,283.56067 h -1.01888"
       id="path65933-9-1-27"
       sodipodi:nodetypes="cc" /><path
       id="rect390-3-5-5-2"
       style="fill:none;stroke:#000000;stroke-width:0.99219;stop-color:#000000"
       d="m 21.449109,248.98097 c 0,-7.08482 5.703647,-12.78847 12.788441,-12.78847 l 33.46728,0.008 c 7.084807,0 12.788444,5.70365 12.788444,12.78846 v 1.05833 c 0,7.0848 -5.703637,12.78845 -12.788444,12.78845 l -33.46728,-0.008 c -7.084794,0 -12.788441,-5.70364 -12.788441,-12.78844 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="AE-architecture-1.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><g
       id="g1378-5-3"
       transform="translate(-35.419228,99.27633)"><path
         id="rect67505-5-62-7"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 68.242985,153.773 16.198577,3.81001 v 10.47214 l -16.198577,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="rect67505-8-3-9-5"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 102.1581,153.773 -16.198598,3.81001 v 10.47214 l 16.198598,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="use69167-4-1-1-9"
         style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 83.513866,157.51966 h 3.244584 v 10.47216 h -3.244584 z"
         sodipodi:nodetypes="ccccc"
         inkscape:tile-x0="27.512444"
         inkscape:tile-y0="-320.67129" /></g><path
       style="fill:none;stroke:#000000;stroke-width:0.85926;marker-end:url(#TriangleStart-7-5-6-36-2);stop-color:#000000"
       d="m 53.259859,236.20021 h -1.01888"
       id="path65933-9-1-27-2"
       sodipodi:nodetypes="cc" /><g
       id="g4447"
       transform="matrix(0.6,0,0,0.6,-24.55408,65.123723)"
       inkscape:export-filename="autoencoder-symbol.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><g
         id="g1378-5-3-3"
         transform="translate(41.270294,83.772591)"><path
           id="rect67505-5-62-7-6"
           style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
           d="m 68.242985,153.773 16.198577,3.81001 v 10.47214 l -16.198577,3.81 z"
           sodipodi:nodetypes="ccccc" /><path
           id="rect67505-8-3-9-5-1"
           style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
           d="m 102.1581,153.773 -16.198598,3.81001 v 10.47214 l 16.198598,3.81 z"
           sodipodi:nodetypes="ccccc" /><path
           id="use69167-4-1-1-9-2"
           style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
           d="m 83.513866,157.51966 h 3.244584 v 10.47216 h -3.244584 z"
           sodipodi:nodetypes="ccccc"
           inkscape:tile-x0="27.512444"
           inkscape:tile-y0="-320.67129" /></g><path
         style="opacity:1;fill:none;stroke:#000000;stroke-width:0.79375;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart-7-5-6-3)"
         d="m 120.81385,243.63296 11.44402,5.29432"
         id="path1215" /></g><path
       id="rect67505-5-62-7-6-9"
       style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:0.6858;stroke-dasharray:none;stop-color:#000000"
       d="m 53.066944,379.07425 4.427479,7.04852 v 5.22494 l -4.427479,7.04851 z"
       sodipodi:nodetypes="ccccc"
       inkscape:export-filename="Transformer-is-super-compressive.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><path
       id="rect67505-5-62-7-6-9-5"
       style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:0.6858;stroke-dasharray:none;stop-color:#000000"
       d="m 149.38236,379.07425 -4.42748,7.04852 v 5.22494 l 4.42748,7.04851 z"
       sodipodi:nodetypes="ccccc" /><path
       id="use69167-4-1-1-9-2-7"
       style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-dasharray:none;stop-color:#000000"
       d="m 57.46697,386.08476 h 4.592583 v 5.22495 H 57.46697 Z"
       sodipodi:nodetypes="ccccc"
       inkscape:tile-x0="55.085717"
       inkscape:tile-y0="385.82017"
       inkscape:tile-cx="57.646592"
       inkscape:tile-cy="388.69723"
       inkscape:tile-w="5.1217496"
       inkscape:tile-h="5.7541166" /><path
       style="fill:none;stroke:#000000;stroke-width:0.79375;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart-7-5-6-3-9)"
       d="m 44.159969,258.71871 11.444014,5.29432"
       id="path1215-1" /><path
       id="rect390-3-5-5-2-0"
       style="fill:none;stroke:#000000;stroke-width:0.744142;stop-color:#000000"
       d="m 82.088112,470.36361 c 0,-5.31361 4.277735,-9.59135 9.591331,-9.59135 l 25.100457,0.006 c 5.31361,0 9.59134,4.27774 9.59134,9.59134 v 0.79375 c 0,5.3136 -4.27773,9.59134 -9.59134,9.59134 l -25.100457,-0.006 c -5.313596,0 -9.591331,-4.27773 -9.591331,-9.59133 z"
       sodipodi:nodetypes="ccccccccc"
       inkscape:export-filename="RL-with-autoencoder.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192" /><g
       id="g1378-5-3-7"
       transform="matrix(0.75,0,0,0.75,39.436859,358.08513)"><path
         id="rect67505-5-62-7-8"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 68.242985,153.773 16.198577,3.81001 v 10.47214 l -16.198577,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="rect67505-8-3-9-5-6"
         style="fill:#cccccc;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 102.1581,153.773 -16.198598,3.81001 v 10.47214 l 16.198598,3.81 z"
         sodipodi:nodetypes="ccccc" /><path
         id="use69167-4-1-1-9-8"
         style="fill:#ffff00;fill-opacity:1;stroke:#000000;stroke-width:1.143;stroke-dasharray:none;stop-color:#000000"
         d="m 83.513866,157.51966 h 3.244584 v 10.47216 h -3.244584 z"
         sodipodi:nodetypes="ccccc"
         inkscape:tile-x0="27.512444"
         inkscape:tile-y0="-320.67129" /></g><path
       style="fill:none;stroke:#000000;stroke-width:0.644445;marker-end:url(#TriangleStart-7-5-6-36-2-7);stop-color:#000000"
       d="m 105.94617,460.77804 h -0.76416"
       id="path65933-9-1-27-2-8"
       sodipodi:nodetypes="cc" /><path
       style="fill:none;stroke:#000000;stroke-width:0.79375;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart-7-5-6-3-94)"
       d="m 34.834037,302.05874 11.444022,5.29432"
       id="path1215-8" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       id="use7670" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(4.8656621)"
       id="use7672" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(9.7313242)"
       id="use7674" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(14.596986)"
       id="use7676" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(19.462648)"
       id="use7678" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(24.328311)"
       id="use7680" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(29.193973)"
       id="use7682" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(34.059635)"
       id="use7684" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(38.925297)"
       id="use7686" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(43.790959)"
       id="use7688" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(48.656621)"
       id="use7690" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(53.522283)"
       id="use7692" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(58.387945)"
       id="use7694" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(63.253607)"
       id="use7696" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(68.119269)"
       id="use7698" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(72.984932)"
       id="use7700" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(77.850594)"
       id="use7702" /><use
       x="0"
       y="0"
       inkscape:tiled-clone-of="#use69167-4-1-1-9-2-7"
       xlink:href="#use69167-4-1-1-9-2-7"
       transform="translate(82.716256)"
       id="use7704" /><g
       inkscape:label=""
       transform="matrix(0.75,0,0,0.75,100.79475,397.17432)"
       id="g8259"><g
   id="surface1"
   transform="matrix(0.705556,0,0,0.705556,-202.086,-60.9837)">
    <g
   style="fill:#000000;fill-opacity:1"
   id="g32">
      <g
   id="use30"
   transform="translate(285.43701,93.917999)">
        <path
   style="stroke:none"
   d="m 3.296875,2.390625 c 0,-0.03125 0,-0.046875 -0.171875,-0.21875 C 1.890625,0.921875 1.5625,-0.96875 1.5625,-2.5 c 0,-1.734375 0.375,-3.46875 1.609375,-4.703125 0.125,-0.125 0.125,-0.140625 0.125,-0.171875 0,-0.078125 -0.03125,-0.109375 -0.09375,-0.109375 -0.109375,0 -1,0.6875 -1.59375,1.953125 -0.5,1.09375 -0.625,2.203125 -0.625,3.03125 0,0.78125 0.109375,1.984375 0.65625,3.125 C 2.25,1.84375 3.09375,2.5 3.203125,2.5 c 0.0625,0 0.09375,-0.03125 0.09375,-0.109375 z m 0,0"
   id="path61" />
      </g>
    </g>
    <g
   style="fill:#000000;fill-opacity:1"
   id="g38">
      <g
   id="use34"
   transform="translate(289.311,93.917999)">
        <path
   style="stroke:none"
   d="M 4.359375,-0.0625 C 5.90625,-0.640625 7.375,-2.421875 7.375,-4.34375 c 0,-1.609375 -1.0625,-2.6875 -2.546875,-2.6875 -2.140625,0 -4.34375,2.265625 -4.34375,4.59375 0,1.65625 1.125,2.65625 2.5625,2.65625 0.25,0 0.578125,-0.046875 0.96875,-0.15625 -0.03125,0.625 -0.03125,0.640625 -0.03125,0.78125 0,0.3125 0,1.09375 0.828125,1.09375 1.171875,0 1.65625,-1.828125 1.65625,-1.9375 0,-0.0625 -0.0625,-0.09375 -0.109375,-0.09375 -0.078125,0 -0.09375,0.046875 -0.125,0.109375 C 6,0.71875 5.421875,0.96875 5.078125,0.96875 c -0.46875,0 -0.609375,-0.265625 -0.71875,-1.03125 z m -1.875,-0.078125 c -0.78125,-0.3125 -1.125,-1.078125 -1.125,-1.984375 0,-0.6875 0.265625,-2.109375 1.015625,-3.171875 C 3.109375,-6.3125 4.046875,-6.78125 4.78125,-6.78125 5.765625,-6.78125 6.5,-6 6.5,-4.671875 c 0,1 -0.515625,3.34375 -2.1875,4.265625 -0.046875,-0.34375 -0.140625,-1.0625 -0.875,-1.0625 -0.53125,0 -1.015625,0.484375 -1.015625,1.015625 0,0.1875 0.0625,0.296875 0.0625,0.3125 z M 3.09375,-0.03125 c -0.140625,0 -0.453125,0 -0.453125,-0.421875 0,-0.40625 0.375,-0.796875 0.796875,-0.796875 0.421875,0 0.609375,0.234375 0.609375,0.84375 0,0.140625 -0.015625,0.15625 -0.109375,0.203125 C 3.671875,-0.09375 3.375,-0.03125 3.09375,-0.03125 Z m 0,0"
   id="path65" />
      </g>
      <g
   id="use36"
   transform="translate(297.18744,93.917999)">
        <path
   style="stroke:none"
   d="m 2.03125,-0.015625 c 0,-0.65625 -0.25,-1.046875 -0.640625,-1.046875 -0.328125,0 -0.53125,0.25 -0.53125,0.53125 C 0.859375,-0.265625 1.0625,0 1.390625,0 1.5,0 1.640625,-0.046875 1.734375,-0.125 1.765625,-0.15625 1.78125,-0.15625 1.78125,-0.15625 c 0.015625,0 0.015625,0 0.015625,0.140625 0,0.75 -0.34375,1.34375 -0.671875,1.671875 -0.109375,0.109375 -0.109375,0.125 -0.109375,0.15625 0,0.078125 0.046875,0.109375 0.09375,0.109375 0.109375,0 0.921875,-0.765625 0.921875,-1.9375 z m 0,0"
   id="path69" />
      </g>
    </g>
    <g
   style="fill:#000000;fill-opacity:1"
   id="g42">
      <g
   id="use40"
   transform="translate(301.6188,93.917999)">
        <path
   style="stroke:none"
   d="M 5.078125,-4.03125 C 5.0625,-4.0625 5.03125,-4.140625 5.03125,-4.171875 c 0,0 0.171875,-0.140625 0.28125,-0.21875 l 1.75,-1.34375 C 8,-6.4375 8.390625,-6.46875 8.6875,-6.5 8.765625,-6.515625 8.859375,-6.515625 8.859375,-6.703125 8.859375,-6.734375 8.828125,-6.8125 8.75,-6.8125 c -0.21875,0 -0.46875,0.03125 -0.703125,0.03125 -0.359375,0 -0.75,-0.03125 -1.109375,-0.03125 -0.0625,0 -0.1875,0 -0.1875,0.203125 0,0.0625 0.046875,0.09375 0.125,0.109375 0.21875,0.015625 0.296875,0.0625 0.296875,0.203125 0,0.1875 -0.296875,0.421875 -0.359375,0.46875 L 2.9375,-2.84375 3.734375,-6.046875 C 3.8125,-6.40625 3.84375,-6.5 4.5625,-6.5 c 0.25,0 0.34375,0 0.34375,-0.203125 0,-0.078125 -0.078125,-0.109375 -0.140625,-0.109375 -0.28125,0 -1,0.03125 -1.28125,0.03125 C 3.203125,-6.78125 2.5,-6.8125 2.203125,-6.8125 2.140625,-6.8125 2,-6.8125 2,-6.625 2,-6.5 2.09375,-6.5 2.296875,-6.5 c 0.125,0 0.3125,0.015625 0.421875,0.015625 0.15625,0.03125 0.21875,0.046875 0.21875,0.171875 0,0.03125 0,0.0625 -0.03125,0.1875 L 1.5625,-0.78125 c -0.09375,0.390625 -0.109375,0.46875 -0.90625,0.46875 -0.171875,0 -0.28125,0 -0.28125,0.1875 C 0.375,0 0.5,0 0.53125,0 0.8125,0 1.515625,-0.03125 1.796875,-0.03125 2,-0.03125 2.21875,-0.015625 2.4375,-0.015625 2.65625,-0.015625 2.875,0 3.078125,0 3.15625,0 3.28125,0 3.28125,-0.203125 3.28125,-0.3125 3.1875,-0.3125 3,-0.3125 c -0.375,0 -0.640625,0 -0.640625,-0.171875 0,-0.078125 0.046875,-0.296875 0.078125,-0.4375 C 2.578125,-1.453125 2.71875,-1.96875 2.84375,-2.5 l 1.484375,-1.15625 1.15625,2.6875 c 0.125,0.265625 0.125,0.296875 0.125,0.34375 0,0.3125 -0.4375,0.3125 -0.53125,0.3125 -0.109375,0 -0.21875,0 -0.21875,0.203125 C 4.859375,0 4.984375,0 5,0 5.40625,0 5.828125,-0.03125 6.21875,-0.03125 6.4375,-0.03125 6.984375,0 7.203125,0 7.25,0 7.375,0 7.375,-0.203125 7.375,-0.3125 7.265625,-0.3125 7.171875,-0.3125 c -0.40625,0 -0.53125,-0.09375 -0.6875,-0.4375 z m 0,0"
   id="path73" />
      </g>
    </g>
    <g
   style="fill:#000000;fill-opacity:1"
   id="g46">
      <g
   id="use44"
   transform="translate(310.23944,93.917999)">
        <path
   style="stroke:none"
   d="m 2.03125,-0.015625 c 0,-0.65625 -0.25,-1.046875 -0.640625,-1.046875 -0.328125,0 -0.53125,0.25 -0.53125,0.53125 C 0.859375,-0.265625 1.0625,0 1.390625,0 1.5,0 1.640625,-0.046875 1.734375,-0.125 1.765625,-0.15625 1.78125,-0.15625 1.78125,-0.15625 c 0.015625,0 0.015625,0 0.015625,0.140625 0,0.75 -0.34375,1.34375 -0.671875,1.671875 -0.109375,0.109375 -0.109375,0.125 -0.109375,0.15625 0,0.078125 0.046875,0.109375 0.09375,0.109375 0.109375,0 0.921875,-0.765625 0.921875,-1.9375 z m 0,0"
   id="path77" />
      </g>
    </g>
    <g
   style="fill:#000000;fill-opacity:1"
   id="g50">
      <g
   id="use48"
   transform="translate(314.66083,93.917999)">
        <path
   style="stroke:none"
   d="M 6.265625,-5.671875 C 6.75,-6.453125 7.171875,-6.484375 7.53125,-6.5 c 0.125,-0.015625 0.140625,-0.171875 0.140625,-0.1875 0,-0.078125 -0.0625,-0.125 -0.140625,-0.125 -0.25,0 -0.546875,0.03125 -0.8125,0.03125 -0.328125,0 -0.671875,-0.03125 -0.984375,-0.03125 -0.0625,0 -0.1875,0 -0.1875,0.1875 0,0.109375 0.078125,0.125 0.15625,0.125 0.265625,0.015625 0.453125,0.125 0.453125,0.328125 0,0.15625 -0.140625,0.375 -0.140625,0.375 l -3.0625,4.875 -0.6875,-5.28125 C 2.265625,-6.375 2.5,-6.5 2.953125,-6.5 c 0.140625,0 0.25,0 0.25,-0.203125 0,-0.078125 -0.078125,-0.109375 -0.125,-0.109375 -0.40625,0 -0.828125,0.03125 -1.25,0.03125 -0.171875,0 -0.359375,0 -0.546875,0 -0.171875,0 -0.359375,-0.03125 -0.53125,-0.03125 -0.078125,0 -0.1875,0 -0.1875,0.1875 0,0.125 0.078125,0.125 0.25,0.125 0.546875,0 0.5625,0.09375 0.59375,0.34375 L 2.1875,-0.015625 C 2.21875,0.1875 2.265625,0.21875 2.390625,0.21875 c 0.15625,0 0.203125,-0.046875 0.28125,-0.171875 z m 0,0"
   id="path81" />
      </g>
    </g>
    <g
   style="fill:#000000;fill-opacity:1"
   id="g54">
      <g
   id="use52"
   transform="translate(322.689,93.917999)">
        <path
   style="stroke:none"
   d="m 2.875,-2.5 c 0,-0.765625 -0.109375,-1.96875 -0.65625,-3.109375 -0.59375,-1.21875 -1.453125,-1.875 -1.546875,-1.875 -0.0625,0 -0.109375,0.046875 -0.109375,0.109375 0,0.03125 0,0.046875 0.1875,0.234375 0.984375,0.984375 1.546875,2.5625 1.546875,4.640625 0,1.71875 -0.359375,3.46875 -1.59375,4.71875 C 0.5625,2.34375 0.5625,2.359375 0.5625,2.390625 0.5625,2.453125 0.609375,2.5 0.671875,2.5 0.765625,2.5 1.671875,1.8125 2.25,0.546875 2.765625,-0.546875 2.875,-1.65625 2.875,-2.5 Z m 0,0"
   id="path85" />
      </g>
    </g>
  </g>
</g><path
       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
       d="m 89.169633,389.26463 c -1.338962,6.93605 6.193175,11.45512 9.331055,10.71136"
       id="path8391"
       sodipodi:nodetypes="cc" /></g><g
     inkscape:groupmode="layer"
     id="layer2"
     inkscape:label="中文"
     style="display:inline"><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
       x="70.071289"
       y="213.86549"
       id="text4451"><tspan
         sodipodi:role="line"
         id="tspan4449"
         style="stroke-width:0.264583"
         x="70.071289"
         y="213.86549">=   <tspan
   style="font-style:normal;font-variant:normal;font-weight:500;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan6005">auto-encoder</tspan>，</tspan><tspan
         sodipodi:role="line"
         style="stroke-width:0.264583"
         x="70.071289"
         y="221.80299"
         id="tspan4567">     它同时有 <tspan
   style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan4457">压缩</tspan> 和 <tspan
   style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan4459">预测未来</tspan> 的功能</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:100.228;display:inline;stroke-width:0.264583"
       x="89.022324"
       y="233.51257"
       id="text4516"
       transform="translate(0,4.2333336)"><tspan
         x="89.022324"
         y="233.51257"
         id="tspan14979"><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan14975">架构</tspan><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan14977"> #1</tspan> 
</tspan><tspan
         x="89.022324"
         y="241.45007"
         id="tspan14981">认为 AE 模拟 人类思考的规则，</tspan><tspan
         x="89.022324"
         y="249.38757"
         id="tspan14983">thoughts = 自然语言句子，
</tspan><tspan
         x="89.022324"
         y="257.32507"
         id="tspan14985">RL 让思考组成闭环路，
</tspan><tspan
         x="89.022324"
         y="265.26257"
         id="tspan14987">thoughts 描述世界。</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:100.228;display:inline;stroke-width:0.264583"
       x="89.022324"
       y="233.51257"
       id="text4516-4"
       transform="translate(1.0055901,55.823279)"><tspan
         x="89.022324"
         y="233.51257"
         id="tspan14993"><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan14989">架构</tspan><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan14991"> #2</tspan> 
</tspan><tspan
         x="89.022324"
         y="241.45007"
         id="tspan14995">认为 AE = 压缩的世界模型，
</tspan><tspan
         x="89.022324"
         y="249.38757"
         id="tspan14997">thoughts = 隐状态，
</tspan><tspan
         x="89.022324"
         y="257.32507"
         id="tspan14999">RL 帮助 AE 解释世界，
</tspan><tspan
         x="89.022324"
         y="265.26257"
         id="tspan15001">解释世界可以给 RL 奖励。</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:152.798;display:inline;stroke-width:0.264583"
       x="29.544897"
       y="348.30627"
       id="text6288"><tspan
         x="29.544897"
         y="348.30627"
         id="tspan15021"><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15003">为什么我认为</tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15005"> </tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15007">强化学习</tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15009"> </tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15011">能解决</tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15013"> </tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15015">hallucination </tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15017">问题？</tspan><tspan
           style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
           id="tspan15019">
</tspan></tspan><tspan
         x="29.544897"
         y="356.24377"
         id="tspan15023">
</tspan><tspan
         x="29.544897"
         y="364.18127"
         id="tspan15027">Transformer 其实是一个超级「<tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15025">压缩</tspan>」的算法，因为它</tspan><tspan
         x="29.544897"
         y="372.11877"
         id="tspan15031">每层的 <tspan
   style="font-style:italic;font-weight:normal;font-family:'Latin Modern Math';-inkscape-font-specification:'Latin Modern Math'"
   id="tspan15029">Q, K, V</tspan> 矩阵是重复使用的（虽然有很多层）：
</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:152.798;display:inline;stroke-width:0.264583"
       x="29.544897"
       y="348.30627"
       id="text6288-4"
       transform="translate(-0.25642963,64.975447)"><tspan
         x="29.544897"
         y="348.30627"
         id="tspan15033">一个学习机器的参数量是有限的，当它「被迫」预测并</tspan><tspan
         x="29.544897"
         y="356.24377"
         id="tspan15035">重构文本时，它就能学习到文本里抽象的规律。
</tspan><tspan
         x="29.544897"
         y="364.18127"
         id="tspan15039"><tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15037">Occam's razor</tspan> 的意思是：用最简约的规律描述现象，</tspan><tspan
         x="29.544897"
         y="372.11877"
         id="tspan15041">得到的就是真确的知识。
</tspan><tspan
         x="29.544897"
         y="380.05627"
         id="tspan15045">强化学习的特点是：它将逻辑推理的过程放进一个 <tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15043">闭环</tspan></tspan><tspan
         x="29.544897"
         y="387.99377"
         id="tspan15051"><tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15047">路</tspan>，也就是强迫使用同一个<tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15049">转移函数</tspan>：</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:6.35px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:152.798;display:inline;stroke-width:0.264583"
       x="29.544897"
       y="348.30627"
       id="text6288-4-3"
       transform="translate(0.02970567,150.25406)"><tspan
         x="29.544897"
         y="348.30627"
         id="tspan15053">在这种强迫之下，机器学习会用一套简约的规则，解释</tspan><tspan
         x="29.544897"
         y="356.24377"
         id="tspan15055">整个世界五花八门的现象，也就是获得智慧或知识，从</tspan><tspan
         x="29.544897"
         y="364.18127"
         id="tspan15057">而分辨正确与谬误，解决 hallucination 的问题。
</tspan><tspan
         x="29.544897"
         y="372.11877"
         id="tspan15059">事实上 GPT / Transformer 已经在做这件事，虽然它没</tspan><tspan
         x="29.544897"
         y="380.05627"
         id="tspan15061">有 RL 闭环，但其实也是反复地试图解释世界 (文本)。
</tspan><tspan
         x="29.544897"
         y="387.99377"
         id="tspan15065">我的想法是希望 RL 的闭环 <tspan
   style="font-weight:normal;-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan15063">增加规则的重复使用</tspan>，提升</tspan><tspan
         x="29.544897"
         y="395.93127"
         id="tspan15067">模型的「智慧」-- 根据 Occam's razor 的原理。</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.7625px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;stroke-width:0.198437"
       x="24.916916"
       y="396.82901"
       id="text233-5"
       inkscape:export-filename="LLM.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan231-6"
         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi';stroke-width:0.198437"
         x="24.916916"
         y="396.82901">text corpus</tspan></text><image
       width="11.162108"
       height="11.162108"
       preserveAspectRatio="none"
       xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAIAAACx0UUtAAAAA3NCSVQICAjb4U/gAAAKuElEQVR4 nO3dX0hT/x/H8dNvgt9gLQonopRTjOUuwpkWDAoSvWh6k7soSS9UCCQDsYuK/qcX3ZheWKDUJEpM aHijXrQclCBh4QRhOUEdxECcRekCRxt+L/zxu/j9vj89O/rZ5/32vB7X53v2qvP8Otvccd/GxoaS WpFIZHx8fHh4OBgMLiwsLC0tpXjAnudyuQYHBw0Gg+whu2Qjhfx+v8vlkv0n1oXa2tpUXlmhUtRo PB5vaWmRfeH0pbm5OTUXV7RUNPrr1y+HwyH7kunRjRs3UnB9RRPe6Pr6utFolH2x9Kujo0P0JRZN eKNOp1P2ZdI7t9st+ioLJbbRnp4e2RcIFEVRRkZGhF5oofZtCHvtKRKJZGZmCjo5JOvz588lJSWy V2jxL3GnfvHihbiTQ7JKS0unp6dlr9BC1NfRRCJx4sSJQCAg4uSg2ezsrNVqlb0iOaIaDYVCeXl5 Is4MO2E0GhcWFsxms+whSUgTdN5Pnz6pOczlchUWFgrasOd1dXVFo9Gk/pNoNJqfnx8Oh00mk6BV u05Uo6urq9seMzAwcOnSJUED9KCvry/ZRhVFiUaj58+f9/l86enpIlbtOlH/Zvr27dvWB2RlZV24 cEHQo+tEIpHQ9h9OTExcvnx5d8eII6rRbf//PnDgQFqaqK/isC2Px1NXVyd7hSoCX3sC4l6/fn3t 2jXZK7aHRnWtu7v78ePHsldsA43q3a1bt/r6+mSv2AoaBaWhoeHNmzeyV/xfaBQURVFqamrev38v e8U/Q6N7WU5OjvqDKyoqvnz5Im6MZmh0L6utre3o6FB/fGlpaTAYFLdHGzS6l8VisdbW1qQ+SXb8 +PFQKCRskRZodO/r7Oysr69Xf/zJkyfVvJWdMmhUF9xud1lZmcqDf/z4ce7cuVgsJnSSemhUL969 e3fq1CmVB09NTTmdTs0/D7C70KheGAwGr9dbXFys8nifz3fx4kWhk1RCozpiMpk8Ho/6j5J7PB4K b+ijUX2xWCwzMzPqj+/u7r579664PWqgUd2xWCx+v1/98e3t7U+ePBG3Z1toVI+Kioq8Xq/6469f vy7x507QqE6Vl5ePjIyoP76hoWF0dFTcni2gUf1yOp1ut1v98ZWVlVJ+7gSN6lp9fX1Sb+hXVFSk /kYSaFTvWltb79y5o/54u92e4jf00SgobW1tSb2hn5eXF4lExO35L2gUFEVR3G53UreBt9vtKfu5 EzQK/zY4OKj+507C4XBFRUVqfu4Eje5lSd2JxGAwjI6Oqs90cnKyurpa067k4C4Me5nX643FYvF4 XOXxaWlphw4dUn/+0dHRhoaGpF7A0gCN7mVTU1NTU1NCH6Kvr+/gwYOdnZ3iHgLP9bBTXV1dQm8k gUZhFwi9kQQahd0h7psKNAq7Q9xdENEoY3vnt9ZuCY0ylpubK3tCKqBRxm7fvi17QiqgUcacTqce ftc6GuXN7XbX1tbKXiEWGuXNZDK9evVqZGRE/fvs7KDRvcDpdI6NjQn9xbI2m03Wnw6NwvYSicSf P39kPToaBerQKFCHRoE6NArUoVGgDo0CdWgUqEOjQB3Lz9wFg8H5+XnZK3jIzs4uKiqSvWJHmDU6 PT1dVVUVDodlD2HG6/WWl5fLXqERp+f6SCRit9sRqAZS7ne3Wzg1+ujRI9kTGLt69arsCRpxatTn 88mewNjExASp316nHqdG//rrL9kTeEvq9k90cGr07NmzsicwVlZWhkaFe/jwoewJjD179kz2BI04 NWoymRYXF/fwhyIEsdlsfr/farXKHqIRs9dHLRbL2NhYKBT6+fOn7C087N+/v6CggPXdIpg1usli scieAKnD6bke9AmNAnVoFKhDo0AdGgXq0ChQh0aBOjQK1KFRoA6NAnVoFKhDo0AdGgXq0ChQh0aB OjQK1KFRoA6NAnUsPyuiKEowGJQ9gQfuH2ZS2DUaiUSampo8Ho/sIZw4HI7+/n6+HwLj1GgsFsvM zJS9gp+JiYm8vLzl5WWz2Sx7ixacvh9tb2+XPYGxpqYm2RM04tTo6Oio7AmMeTyeRCIhe4UWnBpd X1+XPYG3379/y56gBadGi4uLZU9gzGazmUwm2Su04NRoW1ub7AmMvXz5UvYEjTg1arFYvF6v0WiU PYSfgYGBkpIS2Ss04vTak6Io5eXl4XB4bm5uZmZG4m+rZqS4uDg3N5fpq06bmDWqKIrJZCopKeH7 VQGSxem5HvQJjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0 CtShUaCO38/hb0okEvF4XPYKBtLT02VP2Cl+jfb29vb09MzNzckewsPRo0erq6sfPHjA985kzBo9 ffr05OSk7BWcBAKBQCDQ1dW1srLC9Gsqp+9He3t7Eag20Wj05s2bsldoxKlRvncxoOD58+eyJ2jE qdFIJCJ7AmPRaJTpXyCnRo8dOyZ7AmM5OTlM7wTBqdGHDx/KnsAY37tlcWq0pKSko6ND9gqWmpub 6+vrZa/QiNlrT62trZWVlR8+fBgfH5e9hQe73V5WVlZUVCR7iHbMGlUUxWq1Wq3WK1euyB4CKcLp uR70CY0CdWgUqEOjQB0aBerQKFCHRoE6NArUoVGgDo0CdWgUqEOjQB0aBerQKFCHRoE6NArUoVGg Do0Cdfw+K6IoSigUWllZkb2Ch4yMDIvFInvFjjBrNBQKNTY2+nw+2UM4sdls/f39fD92x6nR1dXV vLw82Sv4CQQCdrt9cXGR6RdUTt+P3r9/X/YExhobG2VP0IhTox8/fpQ9gTGfzxeLxWSv0IJTo+vr 67In8IZGhSsrK5M9gTGHw2EymWSv0IJTo/fu3ZM9gbGnT5/KnqARp0bNZrPf7z98+LDsIfx4vV68 9pQiRUVF379/DwaD8/PzsrfwkJ2dzbfOTcwa3bR5WzLZKyBFOD3Xgz6hUaAOjQJ1aBSoQ6NAHRoF 6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAo UIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdG gTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAnahGjUbj1gesra3F 43FBjw67Kx6Pr62tbX3MtldcM1GNHjlyZOsDlpaWhoaGBD067K6hoaGlpaWtj9n2imuWJui8JpNp 22Nqamrevn1bWFgoaAPsiq9fv3o8nm0PU3PFNdoQY3FxUdRiIGl2dlZQSwKf6202m6CTAzU2m62g oEDQyUU1ajAY6urqBJ0cqKmrqzMYDIJOvm9jY0PQqSORSGZmpqCTAynLy8tms1nQyQW+Pmo2m3t6 esSdH4jo6ekRF6iiCPs30384nU6B60E2p9MpOiGBz/WbYrFYRkZGNBoV+igghdFoXFlZSU9PF/oo wt8LTU9PD4fDDodD9ANBijkcjnA4LDpQRRH/XL8pHo+3tLQI/8NAqrS0tMTj8dTEk6JGN/n9fpfL JfuvF3bE5XL5/f5UZiP8+9H/FYlExsfHh4eHg8HgwsLCtm8Eg1xZWVn5+flWq7WqqurMmTNi/wn/ T/4GEGDlyaTZkrgAAAAASUVORK5CYII= "
       id="image5547"
       x="31.962025"
       y="380.20551" /><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.7625px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;display:inline;stroke-width:0.198437"
       x="154.03362"
       y="396.82901"
       id="text233-5-1"
       inkscape:export-filename="LLM.png"
       inkscape:export-xdpi="192"
       inkscape:export-ydpi="192"><tspan
         sodipodi:role="line"
         id="tspan231-6-4"
         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi';stroke-width:0.198437"
         x="154.03362"
         y="396.82901">text corpus</tspan></text><image
       width="11.162108"
       height="11.162108"
       preserveAspectRatio="none"
       xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAIAAACx0UUtAAAAA3NCSVQICAjb4U/gAAAKuElEQVR4 nO3dX0hT/x/H8dNvgt9gLQonopRTjOUuwpkWDAoSvWh6k7soSS9UCCQDsYuK/qcX3ZheWKDUJEpM aHijXrQclCBh4QRhOUEdxECcRekCRxt+L/zxu/j9vj89O/rZ5/32vB7X53v2qvP8Otvccd/GxoaS WpFIZHx8fHh4OBgMLiwsLC0tpXjAnudyuQYHBw0Gg+whu2Qjhfx+v8vlkv0n1oXa2tpUXlmhUtRo PB5vaWmRfeH0pbm5OTUXV7RUNPrr1y+HwyH7kunRjRs3UnB9RRPe6Pr6utFolH2x9Kujo0P0JRZN eKNOp1P2ZdI7t9st+ioLJbbRnp4e2RcIFEVRRkZGhF5oofZtCHvtKRKJZGZmCjo5JOvz588lJSWy V2jxL3GnfvHihbiTQ7JKS0unp6dlr9BC1NfRRCJx4sSJQCAg4uSg2ezsrNVqlb0iOaIaDYVCeXl5 Is4MO2E0GhcWFsxms+whSUgTdN5Pnz6pOczlchUWFgrasOd1dXVFo9Gk/pNoNJqfnx8Oh00mk6BV u05Uo6urq9seMzAwcOnSJUED9KCvry/ZRhVFiUaj58+f9/l86enpIlbtOlH/Zvr27dvWB2RlZV24 cEHQo+tEIpHQ9h9OTExcvnx5d8eII6rRbf//PnDgQFqaqK/isC2Px1NXVyd7hSoCX3sC4l6/fn3t 2jXZK7aHRnWtu7v78ePHsldsA43q3a1bt/r6+mSv2AoaBaWhoeHNmzeyV/xfaBQURVFqamrev38v e8U/Q6N7WU5OjvqDKyoqvnz5Im6MZmh0L6utre3o6FB/fGlpaTAYFLdHGzS6l8VisdbW1qQ+SXb8 +PFQKCRskRZodO/r7Oysr69Xf/zJkyfVvJWdMmhUF9xud1lZmcqDf/z4ce7cuVgsJnSSemhUL969 e3fq1CmVB09NTTmdTs0/D7C70KheGAwGr9dbXFys8nifz3fx4kWhk1RCozpiMpk8Ho/6j5J7PB4K b+ijUX2xWCwzMzPqj+/u7r579664PWqgUd2xWCx+v1/98e3t7U+ePBG3Z1toVI+Kioq8Xq/6469f vy7x507QqE6Vl5ePjIyoP76hoWF0dFTcni2gUf1yOp1ut1v98ZWVlVJ+7gSN6lp9fX1Sb+hXVFSk /kYSaFTvWltb79y5o/54u92e4jf00SgobW1tSb2hn5eXF4lExO35L2gUFEVR3G53UreBt9vtKfu5 EzQK/zY4OKj+507C4XBFRUVqfu4Eje5lSd2JxGAwjI6Oqs90cnKyurpa067k4C4Me5nX643FYvF4 XOXxaWlphw4dUn/+0dHRhoaGpF7A0gCN7mVTU1NTU1NCH6Kvr+/gwYOdnZ3iHgLP9bBTXV1dQm8k gUZhFwi9kQQahd0h7psKNAq7Q9xdENEoY3vnt9ZuCY0ylpubK3tCKqBRxm7fvi17QiqgUcacTqce ftc6GuXN7XbX1tbKXiEWGuXNZDK9evVqZGRE/fvs7KDRvcDpdI6NjQn9xbI2m03Wnw6NwvYSicSf P39kPToaBerQKFCHRoE6NArUoVGgDo0CdWgUqEOjQB3Lz9wFg8H5+XnZK3jIzs4uKiqSvWJHmDU6 PT1dVVUVDodlD2HG6/WWl5fLXqERp+f6SCRit9sRqAZS7ne3Wzg1+ujRI9kTGLt69arsCRpxatTn 88mewNjExASp316nHqdG//rrL9kTeEvq9k90cGr07NmzsicwVlZWhkaFe/jwoewJjD179kz2BI04 NWoymRYXF/fwhyIEsdlsfr/farXKHqIRs9dHLRbL2NhYKBT6+fOn7C087N+/v6CggPXdIpg1usli scieAKnD6bke9AmNAnVoFKhDo0AdGgXq0ChQh0aBOjQK1KFRoA6NAnVoFKhDo0AdGgXq0ChQh0aB OjQK1KFRoA6NAnUsPyuiKEowGJQ9gQfuH2ZS2DUaiUSampo8Ho/sIZw4HI7+/n6+HwLj1GgsFsvM zJS9gp+JiYm8vLzl5WWz2Sx7ixacvh9tb2+XPYGxpqYm2RM04tTo6Oio7AmMeTyeRCIhe4UWnBpd X1+XPYG3379/y56gBadGi4uLZU9gzGazmUwm2Su04NRoW1ub7AmMvXz5UvYEjTg1arFYvF6v0WiU PYSfgYGBkpIS2Ss04vTak6Io5eXl4XB4bm5uZmZG4m+rZqS4uDg3N5fpq06bmDWqKIrJZCopKeH7 VQGSxem5HvQJjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0 CtShUaCO38/hb0okEvF4XPYKBtLT02VP2Cl+jfb29vb09MzNzckewsPRo0erq6sfPHjA985kzBo9 ffr05OSk7BWcBAKBQCDQ1dW1srLC9Gsqp+9He3t7Eag20Wj05s2bsldoxKlRvncxoOD58+eyJ2jE qdFIJCJ7AmPRaJTpXyCnRo8dOyZ7AmM5OTlM7wTBqdGHDx/KnsAY37tlcWq0pKSko6ND9gqWmpub 6+vrZa/QiNlrT62trZWVlR8+fBgfH5e9hQe73V5WVlZUVCR7iHbMGlUUxWq1Wq3WK1euyB4CKcLp uR70CY0CdWgUqEOjQB0aBerQKFCHRoE6NArUoVGgDo0CdWgUqEOjQB0aBerQKFCHRoE6NArUoVGg Do0Cdfw+K6IoSigUWllZkb2Ch4yMDIvFInvFjjBrNBQKNTY2+nw+2UM4sdls/f39fD92x6nR1dXV vLw82Sv4CQQCdrt9cXGR6RdUTt+P3r9/X/YExhobG2VP0IhTox8/fpQ9gTGfzxeLxWSv0IJTo+vr 67In8IZGhSsrK5M9gTGHw2EymWSv0IJTo/fu3ZM9gbGnT5/KnqARp0bNZrPf7z98+LDsIfx4vV68 9pQiRUVF379/DwaD8/PzsrfwkJ2dzbfOTcwa3bR5WzLZKyBFOD3Xgz6hUaAOjQJ1aBSoQ6NAHRoF 6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAo UIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdG gTo0CtShUaAOjQJ1aBSoQ6NAHRoF6tAoUIdGgTo0CtShUaAOjQJ1aBSoQ6NAnahGjUbj1gesra3F 43FBjw67Kx6Pr62tbX3MtldcM1GNHjlyZOsDlpaWhoaGBD067K6hoaGlpaWtj9n2imuWJui8JpNp 22Nqamrevn1bWFgoaAPsiq9fv3o8nm0PU3PFNdoQY3FxUdRiIGl2dlZQSwKf6202m6CTAzU2m62g oEDQyUU1ajAY6urqBJ0cqKmrqzMYDIJOvm9jY0PQqSORSGZmpqCTAynLy8tms1nQyQW+Pmo2m3t6 esSdH4jo6ekRF6iiCPs30384nU6B60E2p9MpOiGBz/WbYrFYRkZGNBoV+igghdFoXFlZSU9PF/oo wt8LTU9PD4fDDodD9ANBijkcjnA4LDpQRRH/XL8pHo+3tLQI/8NAqrS0tMTj8dTEk6JGN/n9fpfL JfuvF3bE5XL5/f5UZiP8+9H/FYlExsfHh4eHg8HgwsLCtm8Eg1xZWVn5+flWq7WqqurMmTNi/wn/ T/4GEGDlyaTZkrgAAAAASUVORK5CYII= "
       id="image5547-9"
       x="161.07874"
       y="380.20551"
       style="display:inline" /></g><g
     inkscape:groupmode="layer"
     id="layer3"
     inkscape:label="English"
     style="display:none"><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.93889px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;display:inline;stroke-width:0.264583"
       x="69.542122"
       y="208.57382"
       id="text4451-3"><tspan
         sodipodi:role="line"
         id="tspan4449-2"
         style="font-size:4.93889px;stroke-width:0.264583"
         x="69.542122"
         y="208.57382">       <tspan
   style="font-style:normal;font-variant:normal;font-weight:500;font-stretch:normal;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan5412">auto-encoder</tspan>，has 2 functions:</tspan><tspan
         sodipodi:role="line"
         style="font-size:4.93889px;stroke-width:0.264583"
         x="69.542122"
         y="214.74744"
         id="tspan4567-0">=     <tspan
   style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi'"
   id="tspan4457-6">compressing information</tspan> and</tspan><tspan
         sodipodi:role="line"
         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93889px;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi';stroke-width:0.264583"
         x="69.542122"
         y="220.92105"
         id="tspan4995">       predicting the future</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.93889px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:100.228;display:inline;stroke-width:0.264583"
       x="89.022324"
       y="233.51257"
       id="text4516-5"
       transform="translate(-1.5875,7.4083338)"><tspan
         x="89.022324"
         y="233.51257"
         id="tspan15071"><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan15069">Architecture #1</tspan>
</tspan><tspan
         x="89.022324"
         y="239.68619"
         id="tspan15073">AE emulates human thinking
</tspan><tspan
         x="89.022324"
         y="245.85981"
         id="tspan15075">Thoughts = natural-language sentences
</tspan><tspan
         x="89.022324"
         y="252.03342"
         id="tspan15077">RL puts thinking into a closed loop
</tspan><tspan
         x="89.022324"
         y="258.20703"
         id="tspan15079">Thoughts decribe the world</tspan></text><text
       xml:space="preserve"
       style="font-weight:300;font-size:4.93889px;line-height:1.25;font-family:'Alibaba PuHuiTi';-inkscape-font-specification:'Alibaba PuHuiTi, Light';letter-spacing:0px;word-spacing:0px;white-space:pre;inline-size:100.228;display:inline;stroke-width:0.264583"
       x="89.022324"
       y="233.51257"
       id="text4516-4-5"
       transform="translate(-1.1110766,58.998281)"><tspan
         x="89.022324"
         y="233.51257"
         id="tspan15083"><tspan
   style="font-weight:500;-inkscape-font-specification:'Alibaba PuHuiTi Medium';fill:#ff0000"
   id="tspan15081">Architecture #2</tspan>
</tspan><tspan
         x="89.022324"
         y="239.68619"
         id="tspan15085">AE = compressed world model
</tspan><tspan
         x="89.022324"
         y="245.85981"
         id="tspan15087">Thoughts = latent representation
</tspan><tspan
         x="89.022324"
         y="252.03342"
         id="tspan15089">RL helps AE to &quot;explain&quot; the world
</tspan><tspan
         x="89.022324"
         y="258.20703"
         id="tspan15091">Successful explanation gives RL reward</tspan></text></g></svg>