Automatic invoice categorization (#75)

* General updates

* Auto categorize imported invoices

* Signal auto categorized invoices

* Fix update_invoice support only for string map

* Fix build env issues

* Fix js import errors causing issues in LiveView

Dockerfile

@@ -9,10 +9,12 @@ ADD mix.exs mix.lock ./
 RUN mix do deps.get, deps.compile
 
 ADD assets/package.json assets/
+
 RUN cd assets && \
     npm install
 
 ADD . .
+RUN echo "ANALYTICS_URL=https://infin.di.uminho.pt/graphql" >> assets/.env
 
 RUN cd assets/ && \
     npm run deploy && \

assets/js/app.js

@@ -8,8 +8,6 @@ import "./register"
 
 import "./calendar"
 import "./modal"
-import "./charts"
-import "./file"
 
 
 

assets/webpack.config.js

@@ -19,7 +19,9 @@ module.exports = (env, options) => {
       ]
     },
     entry: {
-      'app': glob.sync('./vendor/**/*.js').concat(['./js/app.js'])
+      'app': glob.sync('./vendor/**/*.js').concat(['./js/app.js']),
+      'file': glob.sync('./vendor/**/*.js').concat(['./js/file.js']),
+      'charts': glob.sync('./vendor/**/*.js').concat(['./js/charts.js'])
     },
     output: {
       filename: '[name].js',

config/config.exs

@@ -18,7 +18,7 @@ config :infin, InfinWeb.Endpoint,
   pubsub_server: Infin.PubSub,
   live_view: [signing_salt: "ZtkrwQ6M"],
   pt_invoices_url: "localhost:3000",
-  statistics_url: "localhost:5600/graphql"
+  analytics_url: "localhost:5600/graphql"
 
 # Configures Elixir's Logger
 config :logger, :console,

config/prod.exs

@@ -17,10 +17,16 @@ config :infin, InfinWeb.Endpoint,
     "#{System.get_env("PT_INVOICES_HOST") || "localhost"}:#{
       System.get_env("PT_INVOICES_PORT") || "3000"
     }",
+  analytics_url:
+    "#{System.get_env("ANALYTICS_HOST") || "localhost"}:#{
+      System.get_env("ANALYTICS_PORT") || "5600"
+    }#{System.get_env("ANALYTICS_PATH") || "5600"}",
   pt_sibsapimarket: [
-    url: System.get_env("PT_SIBSAPIMARKET_HOST") ||
+    url:
+      System.get_env("PT_SIBSAPIMARKET_HOST") ||
         "https://site1.sibsapimarket.com:8445/sibs/apimarket-sb",
-    apiKey: System.get_env("PT_SIBSAPIMARKET_APIKEY") ||
+    apiKey:
+      System.get_env("PT_SIBSAPIMARKET_APIKEY") ||
         "073286e4-055b-472b-96ad-7ddd484333ec"
   ]
 

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   analytics:
-    image: repo.treescale.com/infinions/analytics:v1.0.0
+    image: repo.treescale.com/infinions/analytics:v1.0.2
     environment:
       API_PORT: 5600
       DB_NAME: infin_dev
@@ -12,21 +12,37 @@ services:
       DB_HOST: db
       DB_ANALYTICS: 'mongodb://analytics_db:27017/'
       DB_ANALYTICS_HOST: analytics_db
-      DB_ANAKYTICS_PORT: 27017
+      DB_ANALYTICS_PORT: 27017
     ports:
     - 5600:5600
+    restart: always
 
   analytics_db:
     image: mongo
     ports:
     - 27017:27017
+    restart: always
+
+  pt-invoices:
+    image: repo.treescale.com/infinions/pt-invoices:v1.0.0
+    environment:
+      NODE_PORT: 3000
+      NODE_ENV: production
+    ports:
+    - 3000:3000
+    restart: always
 
   db:
     image: postgres
-    restart: always
     environment:
       POSTGRES_DB: infin_dev
       POSTGRES_PASSWORD: infin
       POSTGRES_USER: infin
     ports:
     - 5432:5432
+    restart: always
+    volumes:
+    - pgdata:/var/lib/postgresql/data
+
+volumes:
+  pgdata:

lib/infin/importer.ex

@@ -4,7 +4,10 @@ defmodule Infin.Importer do
   """
 
   import Ecto.Query, warn: false
+
+  alias Infin.Companies.Company
   alias Infin.Invoices
+  alias Infin.Repo
 
   require Logger
 
@@ -20,7 +23,8 @@ defmodule Infin.Importer do
     headers = %{"Content-type" => "application/json"}
 
     case HTTPoison.post(
-           Application.get_env(:infin, InfinWeb.Endpoint)[:pt_invoices_url] <> "/invoices",
+           Application.get_env(:infin, InfinWeb.Endpoint)[:pt_invoices_url] <>
+             "/invoices",
            {:stream, enumerable},
            headers
          ) do
@@ -29,22 +33,82 @@ defmodule Infin.Importer do
           200 ->
             %HTTPoison.Response{body: body} = response
             object = Jason.decode!(body)
-            Invoices.insert_fectched_invoices_pt(object["invoices"], company_id)
-            {:ok, "Invoices imported"}
+            invoices = Map.get(object, "invoices", [])
+            {:ok, result} = get_category_prediction(invoices, company_id)
+
+            categories =
+              Jason.decode!(result.body["data"]["categorize_invoices"])
+
+            if categories != [] do
+              Invoices.insert_fectched_invoices_pt(
+                categorize_invoices(invoices, categories),
+                company_id
+              )
+
+              {:ok, "Invoices imported"}
+            else
+              Invoices.insert_fectched_invoices_pt(invoices, company_id)
+
+              {:ok,
+               "Invoices imported, but insufficient data to automatic categorization"}
+            end
 
           400 ->
             %HTTPoison.Response{body: body} = response
             object = Jason.decode!(body)
             {:error, object["message"]}
 
           _ ->
-            %{service: "pt_invoices", message: response} |> inspect() |> Logger.error
+            %{service: "pt_invoices", message: response}
+            |> inspect()
+            |> Logger.error()
+
             {:error, "Service not available"}
         end
 
       {_, message} ->
-        message |> inspect() |> Logger.error
+        message |> inspect() |> Logger.error()
         {:error, "Service not available"}
     end
   end
+
+  defp get_category_prediction(invoices, company_id) do
+    analytics_url =
+      Application.get_env(:infin, InfinWeb.Endpoint)[:analytics_url]
+
+    parsed_invoices =
+      transform_invoices(invoices, company_id) |> String.slice(1..-2)
+
+    Neuron.Config.set(url: analytics_url)
+
+    Neuron.query(
+      "{ categorize_invoices(invoices: \"{\\\"list\\\": #{parsed_invoices}}\")}"
+    )
+  end
+
+  defp transform_invoices(invoices, company_id) do
+    company = Company |> Repo.get(company_id)
+
+    invoices
+    |> Enum.map(fn i ->
+      %{
+        nif: company.nif,
+        company_seller_name: i["nomeEmitente"],
+        total_value: i["valorTotal"],
+        doc_emission_date: i["dataEmissaoDocumento"]
+      }
+    end)
+    |> Jason.encode!()
+    |> Jason.encode!()
+  end
+
+  defp categorize_invoices(invoices, categories) do
+    invoices
+    |> Enum.with_index(1)
+    |> Enum.map(fn {invoice, index} ->
+      invoice
+      |> Map.put(:category_id, categories[to_string(index)])
+      |> Map.put(:automatic_category, true)
+    end)
+  end
 end

lib/infin/invoices.ex

@@ -33,7 +33,8 @@ defmodule Infin.Invoices do
     query =
       from(i in Invoice,
         where: i.company_id == ^company_id,
-        preload: [:company_seller, :category, :company, company: :categories]
+        preload: [:company_seller, :category, :company, company: :categories],
+        order_by: [desc: :doc_emission_date]
       )
 
     Repo.paginate(query, params)
@@ -122,7 +123,7 @@ defmodule Infin.Invoices do
       unless Companies.get_company_by_nif(to_string(invoice["nifEmitente"])) do
         Companies.create_company(%{
           :nif => to_string(invoice["nifEmitente"]),
-          :name => invoice["nomeEmitente"]
+          :name => HtmlEntities.decode(invoice["nomeEmitente"])
         })
       end
 
@@ -131,7 +132,9 @@ defmodule Infin.Invoices do
         :total_value => invoice["valorTotal"],
         :doc_emission_date => invoice["dataEmissaoDocumento"],
         :company_id => company_id,
-        :company_seller_id => Companies.get_company_by_nif(to_string(invoice["nifEmitente"])).id
+        :company_seller_id => Companies.get_company_by_nif(to_string(invoice["nifEmitente"])).id,
+        :category_id => Map.get(invoice, :category_id),
+        :automatic_category => Map.get(invoice, :automatic_category)
       }
 
       create_invoice(document)
@@ -151,8 +154,12 @@ defmodule Infin.Invoices do
 
   """
   def update_invoice(%Invoice{} = invoice, attrs) do
+    at = attrs
+      |> Map.new(fn {k, v} -> {to_string(k), v} end)
+      |> Map.put("automatic_category", false)
+
     invoice
-    |> Invoice.changeset(attrs)
+    |> Invoice.changeset(at)
     |> Repo.update()
   end
 

lib/infin/invoices/invoice.ex

@@ -30,6 +30,7 @@ defmodule Infin.Invoices.Invoice do
     field :merchant_comm, :boolean
     field :consumer_comm, :boolean
     field :is_foreign, :boolean
+    field :automatic_category, :boolean
 
     belongs_to :company_seller, Infin.Companies.Company, foreign_key: :company_seller_id
     belongs_to :company, Infin.Companies.Company, foreign_key: :company_id
@@ -73,7 +74,8 @@ defmodule Infin.Invoices.Invoice do
       :company_id,
       :company_seller_id,
       :category_id,
-      :pdf_id
+      :pdf_id,
+      :automatic_category
     ])
     |> validate_required([:id_document, :doc_emission_date, :total_value, :company_id])
     |> unique_constraint(:id_document)

lib/infin_web/live/dashboard/dashboard.html.leex

@@ -9,7 +9,7 @@
     <%= live_component(@socket, InfinWeb.DashboardLive.Pending,
       id: "#{@company_id}-pending", company_id: @company_id,
       transactions_page_number: 0, transactions_count: 0,
-      invoices_page_number: 0, invoices_count: 0,
+      invoices_page_number: 0, invoices_count: 0
     ) %>
   </div>
 </div>

lib/infin_web/live/dashboard/graph.ex

@@ -1,5 +1,5 @@
 defmodule InfinWeb.DashboardLive.Graph do
-  use Phoenix.LiveComponent
+  use InfinWeb, :live_component
 
   alias Infin.Companies
 

lib/infin_web/live/dashboard/graph.html.leex

@@ -78,3 +78,5 @@
     </div>
   </div>
 </div>
+
+<script src="<%= Routes.static_path(@socket, "/js/charts.js") %>"></script>

lib/infin_web/router.ex

@@ -93,6 +93,6 @@ defmodule InfinWeb.Router do
     resources "/categories", CategoryController, except: [:index, :edit]
     resources "/incomes", IncomeController
     resources "/costs", CostController
-    
+
   end
 end

lib/infin_web/templates/invoice/form.html.eex

@@ -145,3 +145,5 @@
     </div>
   </div>
 <% end %>
+
+<script src="<%= Routes.static_path(@conn, "/js/file.js") %>"></script>

lib/infin_web/templates/invoice/index.html.eex

@@ -20,13 +20,18 @@
                   <td data-label="doc_emission_date"><%= invoice.doc_emission_date %></td>
                   <td data-label="total_value"><%= get_total_value(invoice) %></td>
                   <td data-label="emitter_tax_id"><%= invoice.company_seller.nif %></td>
-                  <td data-label="emitter_name"><%= invoice.company_seller.name %></td>
+                  <td style="width: 33%" data-label="emitter_name"><%= invoice.company_seller.name %></td>
                   <td data-label="category">
-                    <%= if invoice.category == nil do%>
-                    <% else %>
-                      <span class="tag is-info is-medium">
-                        <%= invoice.category.name %>
-                      </span>
+                    <%= if invoice.category != nil do%>
+                      <%= if invoice.automatic_category do%>
+                        <span class="tag is-warning is-medium">
+                          <%= invoice.category.name %>
+                        </span>
+                      <% else %>
+                        <span class="tag is-info is-medium">
+                          <%= invoice.category.name %>
+                        </span>
+                      <% end %>
                     <% end %>
                   </td>
                   <td class="has-text-centered" data-label="total_value">

mix.exs

@@ -52,7 +52,9 @@ defmodule Infin.MixProject do
       {:ex_machina, "~> 2.4"},
       {:httpoison, "~> 1.7"},
       {:scrivener_ecto, "~> 2.0"},
-      {:arc_ecto, "~> 0.11.3"}
+      {:arc_ecto, "~> 0.11.3"},
+      {:html_entities, "~> 0.5.1"},
+      {:neuron, "~> 5.0.0"}
     ]
   end
 
@@ -67,21 +69,23 @@ defmodule Infin.MixProject do
       setup: ["deps.get", "ecto.setup", "cmd npm install --prefix assets"],
       "ecto.bootstrap": ["ecto.migrate", "run priv/repo/banks.exs"],
       "ecto.setup": [
-        "ecto.create",
-        "ecto.migrate",
-        "run priv/repo/seeds.exs",
-        "run priv/repo/banks.exs"
+        "ecto.create --quiet",
+        "ecto.migrate --quiet",
+        "run priv/repo/banks.exs",
+        "run priv/repo/seeds.exs"
       ],
       "ecto.reset": ["ecto.drop", "ecto.setup"],
       test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"],
       "docker.up": [
         "cmd docker login -u=machadovilaca -p=bf9ebaf2843c4b79bb60daeb1c961cbf repo.treescale.com",
-        "cmd docker-compose up -d"
+        "cmd docker-compose up -d",
+        "setup"
       ],
       "docker.down": ["cmd docker-compose down"],
       "podman.up": [
         "cmd podman login -u=machadovilaca -p=bf9ebaf2843c4b79bb60daeb1c961cbf repo.treescale.com",
-        "cmd podman-compose up -d"
+        "cmd podman-compose up -d",
+        "setup"
       ],
       "podman.down": ["cmd podman-compose down"]
     ]