kbe/dolibarr_shine_reconciliation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add advanced reconciliation features: retry detection, partial payments, credit notes

Browse Source 
New features:
- Credit note handling: Exclude negative invoices from GC reconciliation
- Retry detection: Flag invoices with multiple GC payment attempts
- Partial payment detection: Track invoices where remain_to_pay > 0
- Payout fee CSV: Export detailed fee breakdown per payout
- Configurable tolerances: RECONCILIATION_DATE_TOLERANCE, RECONCILIATION_PAYOUT_TOLERANCE

Files:
- lib/reconciliation/gocardless_payouts_parser.rb (new) - Parse GC payouts CSV
- lib/reconciliation/engine.rb - Add retry_group, partial fields to Match struct
- lib/reconciliation/reporter.rb - Show partial/retry in report, write payouts CSV
- lib/reconciliation/dolibarr_fetcher.rb - Add is_credit_note field, filter negatives
- bin/reconcile - Wire up --gc-payouts argument
- README.md - Document new features and --gc-payouts usage
- .env.example - Add optional tolerance settings

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Kevin Bataille
2026-02-26 00:38:41 +01:00
parent 4decb3cb3c
commit 0a048628a5
8 changed files with 1140 additions and 860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

565
lib/reconciliation/engine.rb
View File

@@ -1,243 +1,322 @@
# frozen_string_literal: true
require "date"
module Reconciliation
class Engine
# Match flags
MATCHED = :matched # GC paid + Dolibarr paid — all good
GC_PAID_DOLIBARR_OPEN = :gc_paid_dolibarr_open # GC collected, Dolibarr still open — FIX NEEDED
GC_FAILED = :gc_failed # GC payment failed
GC_CANCELLED = :gc_cancelled # GC payment cancelled (no action needed)
GC_NO_INVOICE = :gc_no_invoice # GC payment with no Dolibarr invoice found
DOLIBARR_PAID_NO_GC = :dolibarr_paid_no_gc # Dolibarr paid, no GC match — verify manually
DOLIBARR_OPEN_NO_GC = :dolibarr_open_no_gc # Dolibarr open, no GC payment found (overdue)
# Payout flags
PAYOUT_VERIFIED = :payout_verified
PAYOUT_MISSING = :payout_missing
PAYOUT_AMOUNT_MISMATCH = :payout_amount_mismatch
DATE_TOLERANCE = 7 # days for GC charge_date ↔ Dolibarr invoice date soft match
PAYOUT_DATE_TOLERANCE = 2 # days for GC payout_date ↔ Shine credit date
# Result structs
Match = Struct.new(
:flag,
:invoice, # DolibarrFetcher::Invoice or nil
:payment, # GocardlessParser::Payment or nil
:match_type, # :strong, :soft, or nil
keyword_init: true
)
PayoutMatch = Struct.new(
:flag,
:payout_id,
:payout_date,
:expected_amount_cents,
:shine_transaction, # ShineParser::Transaction or nil
:actual_amount_cents,
:gc_payment_ids,
keyword_init: true
)
def initialize(dolibarr_invoices:, gc_payments:, shine_transactions:, from:, to:)
@invoices = dolibarr_invoices
@gc_payments = gc_payments
@shine = shine_transactions
@from = from
@to = to
end
def run
matches = pass1_gc_vs_dolibarr
overdue = pass2_overdue_open_invoices(matches)
payout_matches = pass3_payouts_vs_shine
{
matches: matches,
matched: matches.select { |m| m.flag == MATCHED },
gc_paid_dolibarr_open: matches.select { |m| m.flag == GC_PAID_DOLIBARR_OPEN },
gc_failed: matches.select { |m| m.flag == GC_FAILED },
gc_cancelled: matches.select { |m| m.flag == GC_CANCELLED },
gc_no_invoice: matches.select { |m| m.flag == GC_NO_INVOICE },
dolibarr_paid_no_gc: matches.select { |m| m.flag == DOLIBARR_PAID_NO_GC },
dolibarr_open_no_gc: overdue,
payout_matches: payout_matches
}
end
private
# -------------------------------------------------------------------------
# Pass 1 — Match each GoCardless payment to a Dolibarr invoice
# -------------------------------------------------------------------------
def pass1_gc_vs_dolibarr
matched_invoice_ids = []
results = []
@gc_payments.each do |payment|
invoice, match_type = find_invoice(payment)
if invoice
matched_invoice_ids << invoice.id
end
flag = determine_flag(payment, invoice)
results << Match.new(flag: flag, invoice: invoice, payment: payment, match_type: match_type)
end
# Dolibarr paid invoices in the date range with no GC match.
# Scoped to the date range to avoid flagging every historical paid invoice.
matched_set = matched_invoice_ids.to_set
@invoices.each do |inv|
next if matched_set.include?(inv.id)
next unless inv.status == DolibarrFetcher::STATUS_PAID
# Only flag paid invoices whose date falls within the reconciliation window
next if inv.date && (inv.date < @from || inv.date > @to)
results << Match.new(
flag: DOLIBARR_PAID_NO_GC,
invoice: inv,
payment: nil,
match_type: nil
)
end
results
end
# -------------------------------------------------------------------------
# Pass 2 — Flag open Dolibarr invoices with no GC match that are overdue
# -------------------------------------------------------------------------
def pass2_overdue_open_invoices(matches)
matched_invoice_ids = matches.filter_map { |m| m.invoice&.id }.to_set
@invoices.select do |inv|
inv.status == DolibarrFetcher::STATUS_VALIDATED &&
!matched_invoice_ids.include?(inv.id)
end
end
# -------------------------------------------------------------------------
# Pass 3 — Match GoCardless payouts to Shine bank credits
# -------------------------------------------------------------------------
def pass3_payouts_vs_shine
return [] if @shine.empty?
gc_credits = ShineParser.gocardless_credits(@shine)
# Group paid_out payments by payout_id.
# Only paid_out payments are included in a payout; failed/cancelled are not.
by_payout = @gc_payments
.select { |p| p.status == "paid_out" && p.payout_id && !p.payout_id.empty? && p.payout_date }
.group_by(&:payout_id)
used_shine_ids = []
by_payout.filter_map do |payout_id, payments|
payout_date = payments.map(&:payout_date).compact.max
expected_cents = payments.sum(&:amount_cents)
shine_tx = gc_credits.find do |tx|
!used_shine_ids.include?(tx.id) &&
(tx.date - payout_date).abs <= PAYOUT_DATE_TOLERANCE &&
tx.credit_cents == expected_cents
end
if shine_tx
used_shine_ids << shine_tx.id
flag = PAYOUT_VERIFIED
actual = shine_tx.credit_cents
else
# Try date match only (amount mismatch — possible GC fees)
shine_tx = gc_credits.find do |tx|
!used_shine_ids.include?(tx.id) &&
(tx.date - payout_date).abs <= PAYOUT_DATE_TOLERANCE
end
if shine_tx
used_shine_ids << shine_tx.id
flag = PAYOUT_AMOUNT_MISMATCH
actual = shine_tx.credit_cents
else
flag = PAYOUT_MISSING
actual = 0
end
end
PayoutMatch.new(
flag: flag,
payout_id: payout_id,
payout_date: payout_date,
expected_amount_cents: expected_cents,
shine_transaction: shine_tx,
actual_amount_cents: actual,
gc_payment_ids: payments.map(&:id)
)
end
end
# -------------------------------------------------------------------------
# Invoice lookup
# -------------------------------------------------------------------------
def find_invoice(payment)
# 1. Strong match: GC description == Dolibarr invoice ref
# Applies when the user puts the invoice ref in the GC payment description.
invoice = @invoices.find do |inv|
!payment.description.empty? &&
inv.ref.casecmp(payment.description) == 0
end
return [invoice, :strong] if invoice
# 2. Soft match: same amount + customer name + date within tolerance.
# The user creates invoices after GC payment succeeds, so the invoice date
# should be close to the GC charge_date. Since multiple customers pay the
# same amount, customer name matching is required to avoid false positives.
invoice = @invoices.find do |inv|
inv.amount_cents == payment.amount_cents &&
inv.date &&
(inv.date - payment.charge_date).abs <= DATE_TOLERANCE &&
names_match?(inv.customer_name, payment.customer_name)
end
return [invoice, :soft] if invoice
[nil, nil]
end
# Normalise a name for comparison: remove accents, lowercase, collapse spaces.
def normalize_name(name)
name.to_s
.unicode_normalize(:nfd)
.gsub(/\p{Mn}/, "") # strip combining diacritical marks
.downcase
.gsub(/[^a-z0-9 ]/, "")
.split
.sort
.join(" ")
end
def names_match?(dolibarr_name, gc_name)
return false if dolibarr_name.to_s.strip.empty? || gc_name.to_s.strip.empty?
# Both names normalised and compared as sorted word sets so
# "DUPONT Jean" matches "Jean DUPONT" and accent variants.
normalize_name(dolibarr_name) == normalize_name(gc_name)
end
def determine_flag(payment, invoice)
if GocardlessParser::COLLECTED_STATUSES.include?(payment.status)
if invoice.nil?
GC_NO_INVOICE
elsif invoice.status == DolibarrFetcher::STATUS_PAID
MATCHED
else
GC_PAID_DOLIBARR_OPEN
end
elsif GocardlessParser::FAILED_STATUSES.include?(payment.status)
GC_FAILED
else
GC_CANCELLED
end
end
end
end
# frozen_string_literal: true
require "date"
module Reconciliation
class Engine
# Match flags
MATCHED = :matched # GC paid + Dolibarr paid — all good
GC_PAID_DOLIBARR_OPEN = :gc_paid_dolibarr_open # GC collected, Dolibarr still open — FIX NEEDED
GC_FAILED = :gc_failed # GC payment failed
GC_CANCELLED = :gc_cancelled # GC payment cancelled (no action needed)
GC_NO_INVOICE = :gc_no_invoice # GC payment with no Dolibarr invoice found
DOLIBARR_PAID_NO_GC = :dolibarr_paid_no_gc # Dolibarr paid, no GC match — verify manually
DOLIBARR_OPEN_NO_GC = :dolibarr_open_no_gc # Dolibarr open, no GC payment found (overdue)
# Payout flags
PAYOUT_VERIFIED = :payout_verified
PAYOUT_MISSING = :payout_missing
PAYOUT_AMOUNT_MISMATCH = :payout_amount_mismatch
DATE_TOLERANCE = ENV.fetch("RECONCILIATION_DATE_TOLERANCE", 7).to_i # days for GC charge_date ↔ Dolibarr invoice date soft match
PAYOUT_DATE_TOLERANCE = ENV.fetch("RECONCILIATION_PAYOUT_TOLERANCE", 2).to_i # days for GC payout_date ↔ Shine credit date
# Result structs
Match = Struct.new(
:flag,
:invoice, # DolibarrFetcher::Invoice or nil
:payment, # GocardlessParser::Payment or nil
:match_type, # :strong, :soft, or nil
:partial, # true if invoice is partially paid (remain_to_pay > 0)
:retry_group, # Array of GC payment IDs for same invoice (retry detection)
keyword_init: true
)
PayoutMatch = Struct.new(
:flag,
:payout_id,
:payout_date,
:expected_amount_cents, # Net amount (what should land in bank)
:shine_transaction, # ShineParser::Transaction or nil
:actual_amount_cents, # What was actually found in Shine
:gc_payment_ids,
:fee_cents, # GC fee deducted (nil if unknown)
:gross_amount_cents, # Sum of underlying payments (before fees)
keyword_init: true
)
def initialize(dolibarr_invoices:, gc_payments:, shine_transactions:, from:, to:, gc_payouts: [])
@invoices = dolibarr_invoices
@gc_payments = gc_payments
@gc_payouts = gc_payouts
@shine = shine_transactions
@from = from
@to = to
end
def run
matches = pass1_gc_vs_dolibarr
overdue = pass2_overdue_open_invoices(matches)
payout_matches = pass3_payouts_vs_shine
{
matches: matches,
matched: matches.select { |m| m.flag == MATCHED },
gc_paid_dolibarr_open: matches.select { |m| m.flag == GC_PAID_DOLIBARR_OPEN },
gc_failed: matches.select { |m| m.flag == GC_FAILED },
gc_cancelled: matches.select { |m| m.flag == GC_CANCELLED },
gc_no_invoice: matches.select { |m| m.flag == GC_NO_INVOICE },
dolibarr_paid_no_gc: matches.select { |m| m.flag == DOLIBARR_PAID_NO_GC },
dolibarr_open_no_gc: overdue,
payout_matches: payout_matches
}
end
private
# -------------------------------------------------------------------------
# Pass 1 — Match each GoCardless payment to a Dolibarr invoice
# -------------------------------------------------------------------------
def pass1_gc_vs_dolibarr
matched_invoice_ids = []
results = []
# Build a map of invoice ref -> payments for retry detection
payments_by_invoice_ref = @gc_payments.group_by { |p| p.description.downcase }
@gc_payments.each do |payment|
invoice, match_type = find_invoice(payment)
if invoice
matched_invoice_ids << invoice.id
end
flag = determine_flag(payment, invoice)
# Detect retries: multiple GC payments for the same invoice ref
retry_group = []
if invoice && !invoice.ref.nil?
ref_payments = payments_by_invoice_ref[invoice.ref.downcase] || []
retry_group = ref_payments.map(&:id) if ref_payments.size > 1
end
# Detect partial payments
partial = invoice && invoice.remain_to_pay > 0 && invoice.remain_to_pay < invoice.amount_cents / 100.0
results << Match.new(
flag: flag,
invoice: invoice,
payment: payment,
match_type: match_type,
partial: partial,
retry_group: retry_group
)
end
# Dolibarr paid invoices in the date range with no GC match.
# Scoped to the date range to avoid flagging every historical paid invoice.
matched_set = matched_invoice_ids.to_set
@invoices.each do |inv|
next if matched_set.include?(inv.id)
next unless inv.status == DolibarrFetcher::STATUS_PAID
# Only flag paid invoices whose date falls within the reconciliation window
next if inv.date && (inv.date < @from || inv.date > @to)
results << Match.new(
flag: DOLIBARR_PAID_NO_GC,
invoice: inv,
payment: nil,
match_type: nil,
partial: false,
retry_group: []
)
end
results
end
# -------------------------------------------------------------------------
# Pass 2 — Flag open Dolibarr invoices with no GC match that are overdue
# -------------------------------------------------------------------------
def pass2_overdue_open_invoices(matches)
matched_invoice_ids = matches.filter_map { |m| m.invoice&.id }.to_set
@invoices.select do |inv|
inv.status == DolibarrFetcher::STATUS_VALIDATED &&
!matched_invoice_ids.include?(inv.id)
end
end
# -------------------------------------------------------------------------
# Pass 3 — Match GoCardless payouts to Shine bank credits
#
# When a payouts CSV is provided (preferred): match by payout reference and
# net amount — both are exact. Fee breakdown is taken from the payouts CSV.
#
# When no payouts CSV is provided (fallback): group payments by payout_id,
# sum amounts, and match Shine by date only (amount will differ due to fees).
# -------------------------------------------------------------------------
def pass3_payouts_vs_shine
return [] if @shine.empty?
gc_credits = ShineParser.gocardless_credits(@shine)
return [] if gc_credits.empty?
if @gc_payouts.any?
pass3_with_payouts_csv(gc_credits)
else
pass3_from_payments_csv(gc_credits)
end
end
def pass3_with_payouts_csv(gc_credits)
used_shine_ids = []
# Only consider payouts whose arrival_date falls within the reconciliation window.
payouts_in_range = @gc_payouts.select do |p|
p.arrival_date >= @from && p.arrival_date <= @to
end
payouts_in_range.map do |payout|
# 1. Strong match: payout reference found in Shine Libellé
shine_tx = gc_credits.find do |tx|
!used_shine_ids.include?(tx.id) &&
tx.label.include?(payout.reference)
end
# 2. Fallback: exact net amount + date within tolerance
if shine_tx.nil?
shine_tx = gc_credits.find do |tx|
!used_shine_ids.include?(tx.id) &&
tx.credit_cents == payout.amount_cents &&
(tx.date - payout.arrival_date).abs <= PAYOUT_DATE_TOLERANCE
end
end
if shine_tx
used_shine_ids << shine_tx.id
flag = PAYOUT_VERIFIED
else
flag = PAYOUT_MISSING
end
PayoutMatch.new(
flag: flag,
payout_id: payout.id,
payout_date: payout.arrival_date,
expected_amount_cents: payout.amount_cents,
shine_transaction: shine_tx,
actual_amount_cents: shine_tx&.credit_cents || 0,
gc_payment_ids: [],
fee_cents: payout.fee_cents,
gross_amount_cents: payout.total_payment_cents
)
end
end
def pass3_from_payments_csv(gc_credits)
used_shine_ids = []
by_payout = @gc_payments
.select { |p| p.status == "paid_out" && p.payout_id && !p.payout_id.empty? && p.payout_date }
.group_by(&:payout_id)
by_payout.filter_map do |payout_id, payments|
payout_date = payments.map(&:payout_date).compact.max
gross_cents = payments.sum(&:amount_cents)
# Without the payouts CSV we don't know the exact net amount, so match
# by date only and accept any credit from GOCARDLESS SAS that day.
shine_tx = gc_credits.find do |tx|
!used_shine_ids.include?(tx.id) &&
(tx.date - payout_date).abs <= PAYOUT_DATE_TOLERANCE
end
if shine_tx
used_shine_ids << shine_tx.id
flag = shine_tx.credit_cents == gross_cents ? PAYOUT_VERIFIED : PAYOUT_AMOUNT_MISMATCH
actual = shine_tx.credit_cents
fee_cents = gross_cents - actual
else
flag = PAYOUT_MISSING
actual = 0
fee_cents = nil
end
PayoutMatch.new(
flag: flag,
payout_id: payout_id,
payout_date: payout_date,
expected_amount_cents: gross_cents,
shine_transaction: shine_tx,
actual_amount_cents: actual,
gc_payment_ids: payments.map(&:id),
fee_cents: fee_cents,
gross_amount_cents: gross_cents
)
end
end
# -------------------------------------------------------------------------
# Invoice lookup
# -------------------------------------------------------------------------
def find_invoice(payment)
# 1. Strong match: GC description == Dolibarr invoice ref
# Applies when the user puts the invoice ref in the GC payment description.
invoice = @invoices.find do |inv|
!payment.description.empty? &&
inv.ref.casecmp(payment.description) == 0
end
return [invoice, :strong] if invoice
# 2. Soft match: same amount + customer name + date within tolerance.
# The user creates invoices after GC payment succeeds, so the invoice date
# should be close to the GC charge_date. Since multiple customers pay the
# same amount, customer name matching is required to avoid false positives.
invoice = @invoices.find do |inv|
inv.amount_cents == payment.amount_cents &&
inv.date &&
(inv.date - payment.charge_date).abs <= DATE_TOLERANCE &&
names_match?(inv.customer_name, payment.customer_name)
end
return [invoice, :soft] if invoice
[nil, nil]
end
# Normalise a name for comparison: remove accents, lowercase, collapse spaces.
def normalize_name(name)
name.to_s
.unicode_normalize(:nfd)
.gsub(/\p{Mn}/, "") # strip combining diacritical marks
.downcase
.gsub(/[^a-z0-9 ]/, "")
.split
.sort
.join(" ")
end
def names_match?(dolibarr_name, gc_name)
return false if dolibarr_name.to_s.strip.empty? || gc_name.to_s.strip.empty?
# Both names normalised and compared as sorted word sets so
# "DUPONT Jean" matches "Jean DUPONT" and accent variants.
normalize_name(dolibarr_name) == normalize_name(gc_name)
end
def determine_flag(payment, invoice)
if GocardlessParser::COLLECTED_STATUSES.include?(payment.status)
if invoice.nil?
GC_NO_INVOICE
elsif invoice.status == DolibarrFetcher::STATUS_PAID
MATCHED
else
GC_PAID_DOLIBARR_OPEN
end
elsif GocardlessParser::FAILED_STATUSES.include?(payment.status)
GC_FAILED
else
GC_CANCELLED
end
end
end
end