inital upload

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+model-40 filter=lfs diff=lfs merge=lfs -text

WB_sRGB/LICENSE.md

@@ -0,0 +1,438 @@
+Attribution-NonCommercial-ShareAlike 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
+Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-ShareAlike 4.0 International Public License
+("Public License"). To the extent this Public License may be
+interpreted as a contract, You are granted the Licensed Rights in
+consideration of Your acceptance of these terms and conditions, and the
+Licensor grants You such rights in consideration of benefits the
+Licensor receives from making the Licensed Material available under
+these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. BY-NC-SA Compatible License means a license listed at
+     creativecommons.org/compatiblelicenses, approved by Creative
+     Commons as essentially the equivalent of this Public License.
+
+  d. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  e. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  f. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  g. License Elements means the license attributes listed in the name
+     of a Creative Commons Public License. The License Elements of this
+     Public License are Attribution, NonCommercial, and ShareAlike.
+
+  h. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  i. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  j. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  k. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  l. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  m. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  n. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. Additional offer from the Licensor -- Adapted Material.
+               Every recipient of Adapted Material from You
+               automatically receives an offer from the Licensor to
+               exercise the Licensed Rights in the Adapted Material
+               under the conditions of the Adapter's License You apply.
+
+            c. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+  b. ShareAlike.
+
+     In addition to the conditions in Section 3(a), if You Share
+     Adapted Material You produce, the following conditions also apply.
+
+       1. The Adapter's License You apply must be a Creative Commons
+          license with the same License Elements, this version or
+          later, or a BY-NC-SA Compatible License.
+
+       2. You must include the text of, or the URI or hyperlink to, the
+          Adapter's License You apply. You may satisfy this condition
+          in any reasonable manner based on the medium, means, and
+          context in which You Share Adapted Material.
+
+       3. You may not offer or impose any additional or different terms
+          or conditions on, or apply any Effective Technological
+          Measures to, Adapted Material that restrict exercise of the
+          rights granted under the Adapter's License You apply.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material,
+     including for purposes of Section 3(b); and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+

WB_sRGB/classes/WBsRGB.py

@@ -0,0 +1,164 @@
+## White-balance model class
+#
+# Copyright (c) 2018-present, Mahmoud Afifi
+# York University, Canada
+# mafifi@eecs.yorku.ca | m.3afifi@gmail.com
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# All rights reserved.
+#
+# Please cite the following work if this program is used:
+# Mahmoud Afifi, Brian Price, Scott Cohen, and Michael S. Brown,
+# "When color constancy goes wrong: Correcting improperly white-balanced
+# images", CVPR 2019.
+#
+##########################################################################
+
+
+import numpy as np
+import numpy.matlib
+import cv2
+
+
+class WBsRGB:
+  def __init__(self, gamut_mapping=2, upgraded=0):
+    if upgraded == 1:
+      self.features = np.load('WB_sRGB/models/features+.npy')  # encoded features
+      self.mappingFuncs = np.load('WB_sRGB/models/mappingFuncs+.npy')  # correct funcs
+      self.encoderWeights = np.load('WB_sRGB/models/encoderWeights+.npy')  # PCA matrix
+      self.encoderBias = np.load('WB_sRGB/models/encoderBias+.npy')  # PCA bias
+      self.K = 75  # K value for NN searching
+    else:
+      self.features = np.load('WB_sRGB/models/features.npy')  # encoded features
+      self.mappingFuncs = np.load('WB_sRGB/models/mappingFuncs.npy')  # correction funcs
+      self.encoderWeights = np.load('WB_sRGB/models/encoderWeights.npy')  # PCA matrix
+      self.encoderBias = np.load('WB_sRGB/models/encoderBias.npy')  # PCA bias
+      self.K = 25  # K value for nearest neighbor searching
+
+    self.sigma = 0.25  # fall-off factor for KNN blending
+    self.h = 60  # histogram bin width
+    # our results reported with gamut_mapping=2, however gamut_mapping=1
+    # gives more compelling results with over-saturated examples.
+    self.gamut_mapping = gamut_mapping  # options: 1 scaling, 2 clipping
+
+  def encode(self, hist):
+    """ Generates a compacted feature of a given RGB-uv histogram tensor."""
+    histR_reshaped = np.reshape(np.transpose(hist[:, :, 0]),
+                                (1, int(hist.size / 3)), order="F")
+    histG_reshaped = np.reshape(np.transpose(hist[:, :, 1]),
+                                (1, int(hist.size / 3)), order="F")
+    histB_reshaped = np.reshape(np.transpose(hist[:, :, 2]),
+                                (1, int(hist.size / 3)), order="F")
+    hist_reshaped = np.append(histR_reshaped,
+                              [histG_reshaped, histB_reshaped])
+    feature = np.dot(hist_reshaped - self.encoderBias.transpose(),
+                     self.encoderWeights)
+    return feature
+
+  def rgb_uv_hist(self, I):
+    """ Computes an RGB-uv histogram tensor. """
+    sz = np.shape(I)  # get size of current image
+    if sz[0] * sz[1] > 202500:  # resize if it is larger than 450*450
+      factor = np.sqrt(202500 / (sz[0] * sz[1]))  # rescale factor
+      newH = int(np.floor(sz[0] * factor))
+      newW = int(np.floor(sz[1] * factor))
+      I = cv2.resize(I, (newW, newH), interpolation=cv2.INTER_NEAREST)
+    I_reshaped = I[(I > 0).all(axis=2)]
+    eps = 6.4 / self.h
+    hist = np.zeros((self.h, self.h, 3))  # histogram will be stored here
+    Iy = np.linalg.norm(I_reshaped, axis=1)  # intensity vector
+    for i in range(3):  # for each histogram layer, do
+      r = []  # excluded channels will be stored here
+      for j in range(3):  # for each color channel do
+        if j != i:
+          r.append(j)
+      Iu = np.log(I_reshaped[:, i] / I_reshaped[:, r[1]])
+      Iv = np.log(I_reshaped[:, i] / I_reshaped[:, r[0]])
+      hist[:, :, i], _, _ = np.histogram2d(
+        Iu, Iv, bins=self.h, range=((-3.2 - eps / 2, 3.2 - eps / 2),) * 2, weights=Iy)
+      norm_ = hist[:, :, i].sum()
+      hist[:, :, i] = np.sqrt(hist[:, :, i] / norm_)  # (hist/norm)^(1/2)
+    return hist
+
+  def correctImage(self, I):
+    """ White balance a given image I. """
+    # I = I[..., ::-1]  # convert from BGR to RGB  #donna
+    I = im2double(I)  # convert to double
+    # Convert I to float32 may speed up the process.
+    feature = self.encode(self.rgb_uv_hist(I))
+    # Do
+    # ```python
+    # feature_diff = self.features - feature
+    # D_sq = np.einsum('ij,ij->i', feature_diff, feature_diff)[:, None]
+    # ```
+    D_sq = np.einsum(
+      'ij, ij ->i', self.features, self.features)[:, None] + np.einsum(
+      'ij, ij ->i', feature, feature) - 2 * self.features.dot(feature.T)
+
+    # get smallest K distances
+    idH = D_sq.argpartition(self.K, axis=0)[:self.K]
+    mappingFuncs = np.squeeze(self.mappingFuncs[idH, :])
+    dH = np.sqrt(
+      np.take_along_axis(D_sq, idH, axis=0))
+    weightsH = np.exp(-(np.power(dH, 2)) /
+                      (2 * np.power(self.sigma, 2)))  # compute weights
+    weightsH = weightsH / sum(weightsH)  # normalize blending weights
+    mf = sum(np.matlib.repmat(weightsH, 1, 33) *
+             mappingFuncs, 0)  # compute the mapping function
+    mf = mf.reshape(11, 3, order="F")  # reshape it to be 9 * 3
+    I_corr = self.colorCorrection(I, mf)  # apply it!
+    return I_corr
+
+  def colorCorrection(self, input, m):
+    """ Applies a mapping function m to a given input image. """
+    sz = np.shape(input)  # get size of input image
+    I_reshaped = np.reshape(input, (int(input.size / 3), 3), order="F")
+    kernel_out = kernelP(I_reshaped)
+    out = np.dot(kernel_out, m)
+    if self.gamut_mapping == 1:
+      # scaling based on input image energy
+      out = normScaling(I_reshaped, out)
+    elif self.gamut_mapping == 2:
+      # clip out-of-gamut pixels
+      out = outOfGamutClipping(out)
+    else:
+      raise Exception('Wrong gamut_mapping value')
+    # reshape output image back to the original image shape
+    out = out.reshape(sz[0], sz[1], sz[2], order="F")
+    out = out.astype('float32') #donna
+    # out = out.astype('float32')[..., ::-1]  # convert from BGR to RGB
+    return out
+
+
+def normScaling(I, I_corr):
+  """ Scales each pixel based on original image energy. """
+  norm_I_corr = np.sqrt(np.sum(np.power(I_corr, 2), 1))
+  inds = norm_I_corr != 0
+  norm_I_corr = norm_I_corr[inds]
+  norm_I = np.sqrt(np.sum(np.power(I[inds, :], 2), 1))
+  I_corr[inds, :] = I_corr[inds, :] / np.tile(
+    norm_I_corr[:, np.newaxis], 3) * np.tile(norm_I[:, np.newaxis], 3)
+  return I_corr
+
+
+def kernelP(rgb):
+  """ Kernel function: kernel(r, g, b) -> (r,g,b,rg,rb,gb,r^2,g^2,b^2,rgb,1)
+        Ref: Hong, et al., "A study of digital camera colorimetric
+          characterization based on polynomial modeling." Color Research &
+          Application, 2001. """
+  r, g, b = np.split(rgb, 3, axis=1)
+  return np.concatenate(
+    [rgb, r * g, r * b, g * b, rgb ** 2, r * g * b, np.ones_like(r)], axis=1)
+
+
+def outOfGamutClipping(I):
+  """ Clips out-of-gamut pixels. """
+  I[I > 1] = 1  # any pixel is higher than 1, clip it to 1
+  I[I < 0] = 0  # any pixel is below 0, clip it to 0
+  return I
+
+
+def im2double(im):
+  """ Returns a double image [0,1] of the uint8 im [0,255]. """
+  return cv2.normalize(im.astype('float'), None, 0.0, 1.0, cv2.NORM_MINMAX)

WB_sRGB/models/encoderBias.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d77ec0baab6e45a5602cf9f394e7fde3720b8ca76f59c45d7807e0c49b7070
+size 43328

WB_sRGB/models/encoderWeights.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff8e684dd18f0e82fb5b1f398b26eb356f7ba9acd8c81dc43e791f86f105659a
+size 2376128

WB_sRGB/models/features.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ec7f16aaf1cda53dfb1b3812fae88cb56e0157e3fe7e84138227ac1108a9d6
+size 13757828

WB_sRGB/models/mappingFuncs.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d677ecbe0e958ca1f6d56deb9a3ae9ce66cfbb606e891c8e06e60d807de68b6b
+size 8254748

app.py

@@ -0,0 +1,145 @@
+import gradio as gr
+from PIL import Image
+
+from multi_image_process import compute_inp_palette, recolor_single_image, recolor_group_images
+
+
+
+example_images = ["./examples/flower/001.jpg",
+            "./examples/flower/002.jpg",
+            "./examples/flower/003.jpg",
+            "./examples/flower/004.jpg",
+            "./examples/flower/005.jpg",
+            ]
+
+
+
+def swap_to_gallery(images):
+    return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
+
+def remove_back_to_files():
+    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+
+def show_palette(*colors):
+    # create HTML，each color is a block with a border
+    color_blocks = ""
+    for color in colors:
+        if color:  # if choose a color
+            color_blocks += f'<div style="width:40px;height:40px;background:{color};display:inline-block;margin:5px;border:1px solid #000;"></div>'
+    return color_blocks
+
+
+def load_example_images():
+    images = [Image.open(p) for p in example_images]
+    return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(value=example_images, visible=False)
+
+
+
+if __name__ == "__main__":
+
+    with gr.Blocks() as demo:
+        gr.Markdown("# Image recoloring with palette")
+
+
+        # multiple image recoloring for color consistency
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("### Inputs") 
+                image_input = gr.File(
+                            label="Drag (Select) more than one photos",
+                            file_types=["image"],
+                            file_count="multiple"
+                        )
+                uploaded_files = gr.Gallery(label="Input images", visible=False, columns=7, rows=1, height=200)
+
+
+                with gr.Column(visible=False) as clear_button:
+                    remove_and_reupload = gr.ClearButton(value="Remove and upload new ones", components=image_input, size="sm")
+
+
+                image_input.upload(fn=swap_to_gallery, inputs=image_input, outputs=[uploaded_files, clear_button, image_input])
+                remove_and_reupload.click(fn=remove_back_to_files, outputs=[uploaded_files, clear_button, image_input])
+
+                gr.Markdown("### Select the parameters for recoloring")
+                with gr.Row():
+                    with gr.Group(): 
+                        gr.Markdown("### Recoloring without other techiques")
+                        num_center_grp = gr.Dropdown(choices=[1, 2, 3, 4, 5], value=3, label="Number of group palettes")
+                with gr.Group():         
+                    gr.Markdown("### Recoloring with other techiques")
+                    with gr.Row():
+                        with gr.Group(): 
+                            # gr.Markdown("### white balance")
+                            checkbox_input_wb = gr.Checkbox(value=False, label="Apply white balance correction")
+                        with gr.Group(): 
+                            # gr.Markdown("### saliency detection")
+                            checkbox_input_sal = gr.Checkbox(value=False, label="Apply saliency")
+                            checkbox_input_recolor_sal = gr.Checkbox(value=False, label="Recolor salient part only")
+                            checkbox_input_recolor_nonsal = gr.Checkbox(value=False, label="Recolor non-salient part only")
+                            num_center_sal = gr.Dropdown(choices=[1, 2, 3], value=1, label="Number of salient palettes")
+                            num_center_nonsal = gr.Dropdown(choices=[1, 2, 3], value=1, label="Number of non-salient palettes")
+                        with gr.Group(): 
+                            # gr.Markdown("### color naming")
+                            checkbox_input_cn = gr.Checkbox(value=False, label="Apply color naming")
+                            naming_thres = gr.Textbox(value=0.8, label="Threshold of color naming", placeholder=0.8)
+                
+            with gr.Column():
+                gr.Markdown("### Outputs") 
+                output_gallery_palette_in = gr.Gallery(label="Input image palettes", columns=7, rows=1, height=100)
+                output_gallery_palette_group= gr.Gallery(label="Group palette", columns=2, rows=1, height=100)
+                output_gallery_palette_out = gr.Gallery(label="Output image palettes", columns=7, rows=1, height=100)
+                output_gallery_recolor = gr.Gallery(label="Recolored images", columns=7, rows=1, height=300)
+
+        with gr.Row():
+            example_btn = gr.Button("Load Example Images")
+            example_btn.click(fn=load_example_images, outputs=[uploaded_files, clear_button, image_input])
+
+            palette_btn = gr.Button("Compute palette").click(
+                compute_inp_palette, 
+                inputs=[image_input],
+                outputs=[output_gallery_palette_in]
+            )
+
+            # recoloring_multi_btn = gr.Button("Recoloring multiple images").click(
+            #     multi_img_color_consist, 
+            #     inputs=[image_input, num_center_grp, checkbox_input_wb, checkbox_input_sal, checkbox_input_cn, naming_thres],
+            #     outputs=[output_gallery_recolor, output_gallery_palette_in, output_gallery_palette_out, output_gallery_palette_group]
+            # )
+
+            recoloring_multi_btn = gr.Button("Recoloring multiple images").click(
+                recolor_group_images, 
+                inputs=[image_input, num_center_grp, num_center_sal, num_center_nonsal,
+                        checkbox_input_wb, 
+                        checkbox_input_sal, checkbox_input_recolor_sal, checkbox_input_recolor_nonsal,
+                        checkbox_input_cn, naming_thres],
+                outputs=[output_gallery_recolor, output_gallery_palette_in, output_gallery_palette_out, output_gallery_palette_group]
+            )
+
+        
+
+        # # single image recoloring with user-defined palette
+        # with gr.Row():
+        #     with gr.Column():
+        #         gr.Markdown("### Image color categorization")
+        #         with gr.Row():
+        #             color1 = gr.ColorPicker(label="Color 1", value="#ff0000")
+        #             color2 = gr.ColorPicker(label="Color 2", value="#00ff00")
+        #             color3 = gr.ColorPicker(label="Color 3", value="#0000ff")
+        #             color4 = gr.ColorPicker(label="Color 4", value=None)
+        #             color5 = gr.ColorPicker(label="Color 5", value=None)
+        #             palette = gr.HTML()
+        #         with gr.Row():
+        #             btn = gr.Button("Show Picked Palette")
+        #             btn.click(fn=show_palette, inputs=[color1, color2, color3, color4, color5], outputs=palette)
+
+        #     with gr.Column():
+        #         gr.Markdown("### Output image")
+        #         output_gallery_recolor_single = gr.Gallery(label="Recolored image", columns=1, rows=1, height=100)
+        #         recoloring_single_btn = gr.Button("Recoloring single images").click(
+        #             recolor_single_image, 
+        #             inputs=[image_input, color1, color2, color3, color4, color5],
+        #             outputs=[output_gallery_recolor_single]
+        #         )
+
+
+    demo.launch()

color_naming/colornaming.py

@@ -0,0 +1,100 @@
+import numpy as np
+# import cv2
+# from collections import Counter
+from skimage.color import rgb2lab, lab2rgb
+
+
+COLOR_NAME = ['black', 'brown', 'blue', 'gray', 'green', 'orange', 'pink', 'purple', 'red', 'white', 'yellow']
+
+def im2c(img_lab, w2c):
+    """
+    Convert an image to color name representation using a color-name matrix.
+    
+    Returns:
+        numpy.ndarray: Processed image based on color parameter.
+    """
+    img_lab = np.expand_dims(img_lab, axis=0)
+    im = lab2rgb(img_lab)*255
+
+    # Define color name mappings
+    color_values = np.array([[  0,   0,   0], 
+                                [165,  81,  43], 
+                                [  0,   0, 255], 
+                                [127, 127, 127],
+                                [  0, 255,   0], 
+                                [255, 127,   0], 
+                                [255, 165, 216], 
+                                [191,   0, 191],
+                                [255,   0,   0], 
+                                [255, 255, 255], 
+                                [255, 255,   0]], dtype=np.uint8)
+    
+    
+    # Extract RGB channels
+    # RR, GG, BB = im[:, :, 0].flatten(), im[:, :, 1].flatten(), im[:, :, 2].flatten()
+    
+    # Compute index for w2c lookup
+    index_im = ((im[:, :, 0].flatten() // 8) + 32 * (im[:, :, 1].flatten()// 8) + 32 * 32 * (im[:, :, 2].flatten() // 8)).astype(np.int32)
+    
+    # w2cM = np.argmax(w2c, axis=1)
+    # name_idx_img = w2cM[index_im].reshape(im.shape[:2])
+    
+    # max_prob = w2c[np.arange(w2c.shape[0]), w2cM]
+    # max_prob_map = max_prob[index_im].reshape(im.shape[:2])
+    
+    prob_map = w2c[index_im, :].reshape((im.shape[0], im.shape[1], w2c.shape[1]))
+    # max_prob_map = np.max(prob_map, axis=2)
+    name_idx_img = np.argmax(prob_map, axis=2)
+    
+    color_img = np.zeros_like(im).astype(np.uint8)
+    color_nam =[0  for i in range(np.size(im, 1))]
+        
+    for jj in range(im.shape[0]):
+        for ii in range(im.shape[1]):
+            color_img[jj, ii, :] = np.array(color_values[name_idx_img[jj, ii]])
+            color_nam[ii] = COLOR_NAME[name_idx_img[jj, ii]]
+
+    # return prob_map, max_prob_map, name_idx_img, color_img
+    return name_idx_img, color_nam, color_img, prob_map
+
+
+
+
+def compare_color_name(img_src, img_tgt, w2c, threshold=0.9):
+    color_label_org, color_nam_org, _, prob_map_org = im2c(img_src, w2c)
+    color_label_new, color_nam_new, _, prob_map_new = im2c(img_tgt, w2c)
+
+    if threshold==0:
+        is_same_color = (color_label_org==color_label_new)
+    else:
+        diff = np.zeros_like(color_label_org).astype(np.float64)
+        for jj in range(np.size(color_label_org, 0)):
+            for ii in range(np.size(color_label_org, 1)):
+                # difference also can be the l1 , l2 distance, kl divergence between two probablity distribution
+                # diff[jj, ii] = prob_map_org[jj, ii, color_label_org[jj, ii]] - prob_map_new[jj, ii, color_label_org[jj, ii]]
+                diff[jj, ii] = np.linalg.norm(prob_map_org[jj, ii, :]-prob_map_new[jj, ii, :]) 
+        is_same_color = (np.abs(diff) < threshold)
+
+    print(is_same_color)
+    return is_same_color
+
+# if __name__ == "__main__":
+#     w2c = np.load('w2c11_joost_c.npy').astype(np.float16)
+
+
+#     image_path = './test.jpg'
+#     img = cv2.imread(image_path).astype(np.float32)
+#     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+#     prob_map, max_prob_img, name_idx_img, color_img = im2c(img, w2c)
+
+
+#     filtered_counts = Counter(name_idx_img[name_idx_img <= 10])
+#     sorted_counts = sorted(filtered_counts.items(), key=lambda x: x[1], reverse=True)
+#     top_3_values = [num for num, count in sorted_counts[:3]]
+#     top_3_colors = [COLOR_NAME[i] for i in top_3_values]
+
+#     print("Top 3 colors:", top_3_colors)
+
+#     cv2.imwrite('./colormap_joost.jpg', cv2.cvtColor(color_img, cv2.COLOR_BGR2RGB))
+

color_naming/w2c11_joost_c.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc5ede48fed749d552edf1fd5d058266c713f20af3d4a484310103be91766856
+size 2883712

extract_palette.py

@@ -0,0 +1,139 @@
+import numpy as np
+# import pandas as pd
+
+from sklearn.cluster import KMeans
+from sklearn.metrics import pairwise_distances
+
+from collections import Counter
+
+
+
+def histogram(img_lab, bin, mode=2, mask=None):
+    # img_lab = rgb2lab(img_rgb)
+    # img_lab = img_lab.astype(int)
+    if mask is None:
+        mask = np.ones_like(img_lab[:,:,0])
+
+    if img_lab.ndim != 2:
+        img_lab = img_lab.reshape(-1, 3)
+
+    mask = mask.flatten()
+    img_lab_masked = img_lab[mask==1]
+
+    if mode == 3:
+        hist, edges = np.histogramdd(img_lab_masked, bins=bin)
+        xpos, ypos, zpos = np.meshgrid(edges[0][:-1], edges[1][:-1], edges[2][:-1], indexing="ij")
+        hist_samples = np.concatenate((xpos.reshape((bin*bin*bin,1)), ypos.reshape((bin*bin*bin,1)), zpos.reshape((bin*bin*bin,1))), axis=1)
+        hist_counts = hist.reshape(bin*bin*bin)
+
+    elif mode == 2:  
+        hist, xedges, yedges = np.histogram2d(img_lab_masked[:,1], img_lab_masked[:,2], bins=bin, range=None)
+        xpos, ypos = np.meshgrid(xedges[:-1], yedges[:-1], indexing="ij")
+        hist_samples = np.concatenate((xpos.reshape((bin*bin,1)), ypos.reshape((bin*bin,1))), axis=1)
+        hist_counts = hist.reshape(bin*bin)
+    
+    # hist_counts = hist_counts/np.sum(hist_counts)
+
+    return hist_samples, hist_counts
+
+
+
+
+def palette_extraction(img_lab, hist_samples, hist_counts, mode=2, threshold=0.93, num_clusters=5, mask=None):
+
+    if mask is None:
+        mask = np.ones_like(img_lab[:,:,0])
+    
+    if img_lab.ndim != 2:
+        img_lab = img_lab.reshape(-1, 3)
+    
+    mask = mask.flatten()
+    # img_lab = img_lab[mask==1]
+
+
+    hist_densities = hist_counts /np.sum(hist_counts)
+    ###########################     palette extraction    ###########################
+    # inital cluster center
+    index = np.argwhere(hist_densities!=0)
+    index = np.squeeze(index, axis=(1,))
+    num_nonzero = np.size(index)
+
+    # ## directly clustering
+    # num_clusters_opt = num_clusters
+    # kmeans_f = KMeans(n_clusters=num_clusters_opt, init='k-means++', random_state=0).fit(
+    #     hist_samples[index, :], y=None, sample_weight=hist_densities[index])
+    
+
+    ## clustering method from matlab code
+    inits_all = []
+    Cold = np.mean(hist_samples[index, :], 0)
+    distortion=np.zeros((num_clusters,1))
+
+
+    dist = pairwise_distances(hist_samples[index, :], np.expand_dims(Cold, axis=0), metric='euclidean')
+    distortion[0] = np.sum(hist_densities[index] * np.squeeze(dist**2, axis=1), 0)
+
+    inits_all.append(Cold)
+    
+
+
+    for k in range(1, num_clusters):
+        # Initialize the cluster centers
+        k = k+1
+        cinits = np.zeros((k, mode))
+        cw = hist_densities[index]
+        for i in range(k):
+            id = np.argmax(cw)
+            cinits[i,:] = hist_samples[index, :][id,:]
+            d2 = cinits[i,:]* np.ones((num_nonzero, 1)) - hist_samples[index, :]
+            d2 = np.sum(np.square(d2), axis=1)
+            d2 = d2/np.max(d2)
+            cw = cw * (d2**2)
+
+        inits_all.append(cinits)
+        kmeans = KMeans(n_clusters=k, init=cinits, n_init=1).fit(
+                        hist_samples[index, :], y=None, sample_weight=hist_densities[index])
+        
+        dist_point = pairwise_distances(hist_samples[index, :], kmeans.cluster_centers_, metric='euclidean')
+        distortion[k-1] = np.sum(hist_densities[index] * np.min(dist_point, axis=1)**2)
+
+    variance = distortion[:-1] - distortion[1:]
+    distortion_percent = np.cumsum(variance)/(distortion[0]-distortion[-1])
+
+    r=np.argwhere(distortion_percent > threshold)
+    num_clusters_opt = np.min(r)+2
+
+    kmeans_f = KMeans(n_clusters=num_clusters_opt, init=inits_all[num_clusters_opt-1], n_init=1).fit(
+                      hist_samples[index, :], y=None, sample_weight=hist_densities[index])
+    cluster_centers = kmeans_f.cluster_centers_
+
+
+    # print(cluster_centers.shape)
+
+    if mode ==3:
+        img_labels = kmeans_f.predict(img_lab)
+    elif mode == 2:
+        img_labels = kmeans_f.predict(img_lab[:, 1:3])
+
+    hist_labels = kmeans_f.predict(hist_samples)
+    
+
+        # print(cluster_centers.shape)
+
+    # # lab to rgb
+    # cluster_cen_rgb = lab2rgb(np.expand_dims(cluster_centers, axis=0))
+    # cluster_cen_rgb = np.squeeze(cluster_cen_rgb, axis=0)
+    
+    img_labels[mask==0] = 255
+    c_densities = np.zeros(num_clusters_opt)
+    
+    dict=Counter(img_labels)
+    for key in np.unique(img_labels):
+        if key == 255:
+            continue
+        c_densities[key] = dict.get(key)
+
+    c_densities = c_densities / np.sum(c_densities)
+    
+
+    return cluster_centers, c_densities, img_labels, hist_labels

image.py

@@ -0,0 +1,242 @@
+# import cv2
+import os
+import numpy as np
+
+from PIL import Image
+from skimage.color import rgb2lab, lab2rgb, rgb2hsv, hsv2rgb
+from WB_sRGB.classes import WBsRGB as wb_srgb
+from extract_palette import histogram, palette_extraction
+from saliency.LDF.infer import Saliency_LDF
+from saliency.fast_saliency import get_saliency_ft, get_saliency_mbd
+from utils import color_difference
+
+
+class BaseImage:
+    def __init__(self, filepath):
+        self.filename = os.path.basename(filepath.name)
+        self.image = Image.open(filepath)
+        self.img_rgb = np.asarray(self.image).astype(dtype=np.uint8)
+        self.img_lab = rgb2lab(self.img_rgb)
+
+        self.bin_size = 16
+        self.mode = 2
+        self.hist_harmonization = False
+        self.template = 'L'
+        self.distortion_threshold = 0.93
+        self.num_center_ind = 7
+        self.lightness = 70
+        # self.if_correct_wb = if_correct_wb
+        # self.if_saliency = if_saliency
+        # self.saliency_threshold = sal_thres
+        self.cdiff_threshold = 30
+        self.sal_threshold = 0.9
+        self.applied_wb = False
+        # self.valid_class = [0,1]
+
+        
+        # self.hist_value, self.hist_count, \
+        # self.c_center, self.c_density, \
+        # self.c_img_label = self.extract_palette(if_wb=self.if_correct_wb, 
+        #                                         if_saliency=self.if_saliency, 
+        #                                         sal_thres=self.saliency_threshold)
+        
+        # self.inital_info(self.if_correct_wb, 
+        #                  self.if_saliency, 
+        #                  self.saliency_threshold)
+        
+        
+        # self.hist_value, self.hist_count, self.c_center, self.c_density, self.c_img_label = self.extract_palette(if_wb=self.if_correct_wb, if_saliency=False)
+        # self.hist_value_sal, self.hist_count_sal, self.c_center_sal, self.c_density_sal, self.c_img_label_sal = self.extract_palette(if_wb=self.if_correct_wb, if_saliency=True, sal_thres=self.saliency_threshold)
+
+    def inital_info(self, if_correct_wb, if_saliency, wb_thres, sal_thres, valid_class):
+        self.hist_value, self.hist_count, \
+        self.c_center, self.c_density, \
+        self.c_img_label, self.sal_links = self.extract_salient_palette(if_wb=if_correct_wb, 
+                                                        if_saliency=if_saliency,
+                                                        wb_thres=wb_thres, 
+                                                        sal_thres=sal_thres, 
+                                                        valid_class=valid_class)
+        
+        self.label_colored = self.cal_color_segment()
+
+    def get_rgb_image(self):
+        return self.img_rgb
+    
+    def get_lab_image(self):
+        return self.img_lab
+    
+    def get_wb_image(self):
+        self.img_wb = self.white_balance_correction()
+        return self.img_wb
+    
+    def get_saliency(self):
+        self.sal_map = self.saliency_detection(self.img_rgb)
+        return self.sal_map
+    
+    def get_color_segment(self):
+        return self.label_colored
+    
+    def get_label(self):
+        # print(self.links)
+        # label_mapped = np.zeros_like(self.colorlabel)
+        # for id, label in enumerate(self.links):
+        #     label_mapped[self.colorlabel==id] = label
+        # self.colorlabel = label_mapped
+        return self.colorlabel
+    
+
+    def cal_color_segment(self):
+        label_colored = np.zeros_like(self.img_rgb, dtype=np.float64)
+        for id_color in range(np.size(self.center, 0)):
+            label_colored[self.colorlabel == id_color] = self.center[id_color, :]
+        label_colored = lab2rgb(label_colored)
+        label_colored = np.round(label_colored*255).astype(np.uint8)
+        return label_colored
+    
+
+    # def cal_salient_segment(self, palettelabel):
+    #     label_colored = np.zeros_like(self.img_rgb, dtype=np.float64)
+    #     valid_label = np.argwhere(palettelabel==1).flatten()
+    #     for id_color in valid_label:
+    #         label_colored[self.colorlabel == id_color] = self.center[id_color, :] 
+    #     label_colored = lab2rgb(label_colored)
+    #     label_colored = np.round(label_colored*255).astype(np.uint8)
+    #     return label_colored
+
+
+    def white_balance_correction(self):
+        # print('Correcting the white balance...')
+        # use upgraded_model = 1 to load our new model that is upgraded with new
+        # training examples.
+        upgraded_model = 2
+        # use gamut_mapping = 1 for scaling, 2 for clipping (our paper's results
+        # reported using clipping). If the image is over-saturated, scaling is
+        # recommended.
+        gamut_mapping = 2
+        # processing
+        # create an instance of the WB model
+        wbModel = wb_srgb.WBsRGB(gamut_mapping=gamut_mapping,
+                                upgraded=upgraded_model)
+        img_wb = wbModel.correctImage(self.img_rgb)  # white balance it
+        image_wb = (img_wb*255).astype(np.uint8)
+        # img_wb = cv2.cvtColor(img_wb, cv2.COLOR_BGR2RGB)
+        return image_wb
+
+
+    def saliency_detection(self, img_rgb, method='LDF'):
+        if method == 'LDF':
+            get_saliency_LDF = Saliency_LDF()
+            sal_map = get_saliency_LDF.inference(img_rgb)
+        elif method == 'ft':
+            sal_map = get_saliency_ft(img_rgb)
+        elif method == 'rbd':
+            sal_map = get_saliency_mbd(img_rgb)
+        return sal_map
+
+
+    def solve_ind_palette(self, img_rgb, mask_binary=None):
+        w, h, c = img_rgb.shape
+        img_lab = rgb2lab(img_rgb)    # lab transfer by function
+
+        hist_value, hist_count = histogram(img_lab, self.bin_size, mode=self.mode, mask=mask_binary)   ## with numpy histogram 
+    
+        ## extract palette
+        # mask_binary = np.ones_like(self.img_rgb[:,:,0])
+        c_center, c_density, c_img_label, histlabel = palette_extraction(img_lab, hist_value, hist_count, 
+                                                                            threshold=self.distortion_threshold, 
+                                                                            num_clusters=self.num_center_ind,
+                                                                            mode=self.mode,
+                                                                            mask=mask_binary)
+        
+        if self.mode == 2:
+            c_center = np.insert(c_center, 0, values=self.lightness, axis=1)
+
+        c_img_label = np.reshape(c_img_label, (w,h))
+        # density = np.tile(hist_counts, (self.mode, 1))
+
+        return hist_value, hist_count, c_center, c_density, c_img_label, histlabel
+    
+
+
+
+    def extract_salient_palette(self, if_wb=False, if_saliency=False, wb_thres=5, sal_thres=0.9, valid_class=[0,1]):
+
+        img_rgb = self.img_rgb.copy()
+        if if_wb:
+            self.img_wb = self.white_balance_correction()
+            img_wb = self.img_wb
+            dE = color_difference(img_rgb, img_wb)
+            print(dE)
+            if dE > wb_thres:
+                self.applied_wb = True
+                img_rgb = img_wb
+                print('use white balance correction on {}'.format(self.filename.split('/')[-1]))
+
+        hist_value, hist_count, center, density, colorlabel, histlabel  = self.solve_ind_palette(img_rgb, mask_binary=None)
+        self.center = center
+        self.colorlabel = colorlabel
+
+        sal_links = [i for i in range(np.size(center, axis=0))]
+
+        if not if_saliency:
+            return hist_value, hist_count, center, density, colorlabel, sal_links
+        
+        else:
+            self.sal_map = self.saliency_detection(self.img_rgb)
+            label_sem = np.zeros_like(self.img_rgb[:,:,0])
+            # print(label_sem.shape, self.sal_map.shape)
+            label_sem[self.sal_map > sal_thres]=1
+            
+            p_feq = np.zeros((len(valid_class), np.size(center, axis=0)))
+
+
+            for id_cls, cls in enumerate(valid_class):
+                label_binary = np.zeros_like(label_sem)
+                label_binary[label_sem==cls] = 1
+                colorlabel_cls = colorlabel[label_binary==1]
+                value, count = np.unique(colorlabel_cls, return_counts=True)
+                p_feq[id_cls, value] = count/count.sum()
+
+            palettelabel = np.argmax(p_feq, axis=0)
+
+            class_num = len(valid_class)
+            c_center = [np.array([]) for i in range(class_num)]
+            c_density = [np.array([]) for i in range(class_num)]
+            c_img_label = [np.array([]) for i in range(class_num)]
+            hist_samples = [np.array([]) for i in range(class_num)]
+            hist_counts = [np.array([]) for i in range(class_num)]
+            mapping = [np.array([]) for i in range(class_num)]
+
+
+            for id_cls, cls in enumerate(valid_class):
+                mapping[id_cls] = np.argwhere(palettelabel==id_cls).flatten()
+                c_center[id_cls]= center[mapping[id_cls],:]
+                c_density[id_cls] = density[mapping[id_cls]]
+                hist_samples[id_cls] = hist_value.copy()
+                hist_counts[id_cls] = hist_count.copy()
+                hist_counts[id_cls][histlabel!=id_cls] = 0
+                
+
+                for idx, label in enumerate(mapping[id_cls]):
+                    labels = np.zeros_like(colorlabel)
+                    labels[colorlabel==label] = idx
+                c_img_label[id_cls] = labels
+
+                # if id_cls ==1:
+                #     label_colored = np.zeros_like(self.img_rgb, dtype=np.float64)
+                #     for id_color in mapping[id_cls]:
+                #         label_colored[colorlabel == id_color] = center[id_color, :] 
+                #     label_colored = lab2rgb(label_colored)
+                #     label_colored = np.round(label_colored*255).astype(np.uint8)
+
+                # print(colorlabel.shape, c_img_label[id_cls].shape)
+                # print(density.shape, c_density[id_cls].shape)
+                # print(center.shape, c_center[id_cls].shape)
+
+            sal_links = np.hstack((mapping[1], mapping[0]))
+
+            # print(links)
+
+            return hist_samples, hist_counts, c_center, c_density, c_img_label, sal_links
+
+

multi_image_process.py

@@ -0,0 +1,365 @@
+import os
+import cv2
+import numpy as np
+
+from skimage.color import rgb2lab
+
+from solve_group_palette import solve_group_palette
+from recolor import lab_transfer
+from color_naming.colornaming import compare_color_name
+from utils import visualize_palette
+from image import BaseImage
+
+
+# recolor single image with given palette  TO DO
+def recolor_single_image(image, save_dir='./results/testing'):
+    image = image
+
+    return image
+
+
+# compute and save the palette for each image
+def compute_inp_palette(images, save_dir='./results/testing'):
+    palette_all = [i for i in range(len(images))]
+    
+    for img_id, img in enumerate(images):
+        
+        img_name = os.path.basename(img.name)
+        print('processing image {}: {}...'.format(img_id, img_name))
+        image = BaseImage(img)
+        _, _, c_center, _, _, _ = image.solve_ind_palette(image.img_rgb, mask_binary=None)
+        # print(c_center)
+
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+        imwrite_path = os.path.join(save_dir, 'palette_'+ img_name[:-4]+'.png')
+        img_palette = visualize_palette(c_center, patch_size=20)
+        img_palette = np.round(img_palette*255).astype(np.uint8)
+        cv2.imwrite(imwrite_path, cv2.cvtColor(img_palette, cv2.COLOR_RGB2BGR))
+        
+        palette_all[img_id] = img_palette.copy()
+    return palette_all
+
+
+# match the palette with the group palette
+def match_palette(palette_ind, palette_grp, L_idx, if_cn, naming_thres):
+    # print(L_idx)
+    # print(palette_ind)
+    palette_mapped = palette_ind.copy()
+    valid = (L_idx > 0)
+    # print('L_idx:', L_idx)
+    if L_idx.size == 0:
+        return palette_mapped, L_idx
+    valid = valid.flatten()
+    idx = np.argwhere(L_idx.flatten() > 0).flatten()
+
+    palette_mapped[idx, :] = palette_grp[L_idx[valid].flatten()-1, :]
+
+    if if_cn:
+        # print("Check the matching colors with color naming")
+        # print(L_idx, palette_mapped)
+        w2c = np.load('./color_naming/w2c11_joost_c.npy').astype(np.float16)
+        is_the_same = compare_color_name(palette_ind, palette_mapped, w2c, threshold=naming_thres)
+        mask = (is_the_same == True).T
+        idx = np.argwhere(is_the_same == False).flatten()
+        L_idx = L_idx * mask
+        palette_mapped[idx, :] = palette_ind[idx, :]
+        # print(L_idx, palette_mapped)
+    return palette_mapped, L_idx
+
+
+# solve the group palette without saliency
+def solve_grp_palette(images, mode, bin_size, 
+                      if_wb=False, wb_thres=20, num_center=5,
+                      lightness=70., eta=1e10, gamma=0, iteration=10,
+                      if_cn=False, naming_thres=0.5):
+    
+    num_img = len(images)
+    c_center = [np.array([]) for i in range(num_img)]
+    c_density = [np.array([]) for i in range(num_img)]
+    palette_map = [np.array([]) for i in range(num_img)]
+    L_idx = [np.array([]) for i in range(num_img)]
+
+    if mode == 3:
+        hist_samples_all = np.zeros((bin_size**3, 3))
+        hist_counts_all = np.zeros(bin_size**3)
+
+    elif mode ==2:
+        hist_samples_all = np.zeros((bin_size**2, 2))
+        hist_counts_all = np.zeros(bin_size**2)
+
+    for img_id, image in enumerate(images):
+        image.inital_info(if_wb, 
+                          False,
+                          wb_thres,
+                          0.9,
+                          [0, 1])
+        
+        density = np.tile(image.hist_count, (mode, 1))
+        hist_counts_all  = hist_counts_all  + image.hist_count
+        hist_samples_all = hist_samples_all + density.T * image.hist_value
+
+    index = np.argwhere(hist_counts_all != 0)
+    index = np.squeeze(index, axis=(1,))
+
+    hist_counts_all = hist_counts_all[index]
+    hist_samples_all = hist_samples_all[index, :] / np.expand_dims(hist_counts_all, axis=1)
+
+
+    print('Solving the group palette...')
+    ## take the number of palettes of each input image as the reference 
+    reference = np.zeros((num_img, 1))
+    if np.sum(reference) == 0:
+        reference = reference + 1
+
+    num_palettes = 0
+    for i in range(num_img):
+        if reference[i]:
+            num_palettes = num_palettes + np.size(images[i].c_center, 1)
+
+
+    ##### calculate inital group center with kmeans
+    # print(palette_size_default.get().dtype())
+    m = np.minimum(int(num_center), num_palettes)
+
+    c_center = [image.c_center for image in images]
+    c_density = [image.c_density for image in images]
+
+
+    M, matching = solve_group_palette(hist_samples_all, hist_counts_all, 
+                                        c_center, c_density, reference, m, 
+                                        lightness=lightness, eta=eta,
+                                        gamma=gamma, iteration=iteration)
+    
+    for img_id in range(num_img):
+        palette_map[img_id], L_idx[img_id] = match_palette(images[img_id].c_center, M, matching[img_id], if_cn, naming_thres)
+    # print('c_center: ', c_center)
+    
+    return c_center, M, palette_map, L_idx
+
+
+
+# solve the group palette with saliency
+def solve_grp_palette_wsal(images, mode, bin_size,
+                            lightness=70., eta=1e10, gamma=0, iteration=10,
+                            if_wb=False, wb_thres=30, 
+                            if_saliency=True, sal_thres=0.9, valid_class=[0,1], 
+                            sal_center=1, nonsal_center=1, 
+                            recolor_nonsal_only=False, recolor_sal_only=False,
+                            if_cn=False, naming_thres=0.5):
+    
+    class_num = len(valid_class)
+    num_img = len(images)
+
+    c_center = [np.array([]) for i in range(num_img)]
+    c_density = [np.array([]) for i in range(num_img)]
+    
+    M = [np.array([]) for i in range(class_num)]
+    matching = [np.array([]) for i in range(class_num)]
+
+    palette_map = [[np.array([]) for i in range(num_img)] for i in range(class_num)]
+    L_idx = [[np.array([]) for i in range(num_img)] for i in range(class_num)]
+    p_src_sal_size = [np.array([]) for i in range(num_img)]
+
+    if mode == 3:
+        hist_samples_all = [np.zeros((bin_size**3, 3)) for i in range(class_num)]
+        hist_counts_all = [np.zeros(bin_size**3) for i in range(class_num)]
+
+    elif mode ==2:
+        hist_samples_all = [np.zeros((bin_size**2, 2)) for i in range(class_num)]
+        hist_counts_all = [np.zeros(bin_size**2) for i in range(class_num)]
+
+
+    for img_id, image in enumerate(images):
+
+        image.inital_info(if_wb,
+                          if_saliency,
+                          wb_thres,
+                          sal_thres,
+                          valid_class)
+        
+        for id_cls, cls in enumerate(valid_class):
+            density = np.tile(image.hist_count[id_cls], (mode, 1))
+            hist_counts_all[id_cls]  = hist_counts_all[id_cls]  + image.hist_count[id_cls]
+            hist_samples_all[id_cls] = hist_samples_all[id_cls] + density.T * image.hist_value[id_cls]
+
+
+    for id_cls, cls in enumerate(valid_class):
+        index = np.argwhere(hist_counts_all[id_cls] != 0)
+        index = np.squeeze(index, axis=(1,))
+
+        hist_counts_all[id_cls] = hist_counts_all[id_cls][index]
+        hist_samples_all[id_cls] = hist_samples_all[id_cls] [index, :] / np.expand_dims(hist_counts_all[id_cls], axis=1)
+
+
+        print('Solving the group palette...')
+        ## take the number of palettes of each input image as the reference 
+        reference = np.zeros((num_img, 1))
+        if np.sum(reference) == 0:
+            reference = reference + 1
+
+        num_palettes = 0
+        for i in range(num_img):
+            if reference[i] and images[i].c_center[id_cls].size != 0:
+                num_palettes = num_palettes + np.size(images[i].c_center[id_cls] , 1)
+
+
+        ##### calculate inital group center with kmeans
+        # print(palette_size_default.get().dtype())
+        m=[1,1]
+        m[0] = np.minimum(int(sal_center), num_palettes)
+        m[1] = np.minimum(int(nonsal_center), num_palettes)
+        
+
+        c_center = [image.c_center[id_cls]  for image in images]
+        c_density = [image.c_density[id_cls]  for image in images]
+        # print(c_center[id_cls])
+
+
+        M[id_cls], matching[id_cls] = solve_group_palette(hist_samples_all[id_cls], hist_counts_all[id_cls], 
+                                    c_center, c_density, reference, m[id_cls], 
+                                    lightness=lightness, eta=eta,
+                                    gamma=gamma, iteration=iteration)
+        # print(M[id_cls], matching[id_cls])
+        
+        for img_id in range(num_img):
+            palette_map[id_cls][img_id], L_idx[id_cls][img_id] = match_palette(images[img_id].c_center[id_cls], M[id_cls], matching[id_cls][img_id], if_cn, naming_thres)
+            if id_cls == 1:
+                p_src_sal_size[img_id] = np.size(palette_map[1][img_id], axis=0)
+            
+
+
+        # if only_nonsal_var.get():
+        #     palette_map[1][img_id] = c_center[1][img_id]
+
+    p_grp_sal_size = np.size(M[1], axis=0)
+    p_nsal_size = np.size(M[0], axis=0)
+    
+    cls_keep = None
+    if recolor_nonsal_only:
+        print('only recolor non-salient region')
+        cls_keep = 0
+    elif recolor_sal_only:
+        print('only recolor salient region')
+        cls_keep = 1
+        
+    if cls_keep is not None:
+        for img_id in range(num_img):
+            # print(L_idx[1][img_id])
+            # print(palette_map[1][img_id])
+            if  L_idx[cls_keep][img_id].size == 0:
+                palette_map[cls_keep][img_id] = images[img_id].c_center[cls_keep]
+                L_idx[cls_keep][img_id] = 0
+            else:
+                # print(L_idx[cls_keep][img_id])
+                for idx_nsal in range(len(L_idx[cls_keep][img_id])):
+                    palette_map[cls_keep][img_id][idx_nsal,:] = images[img_id].c_center[cls_keep][idx_nsal,:]
+                    L_idx[cls_keep][img_id][idx_nsal] = 0
+
+
+    L_idx_img = [0 for i in range(num_img)]
+    palette_map_img = [0 for i in range(num_img)]
+    center_img = [0 for i in range(num_img)]
+    
+    
+    for img_id in range(num_img):
+        # print(L_idx[0][img_id], L_idx[1][img_id])
+        center_img[img_id] = np.vstack([images[img_id].c_center[1], images[img_id].c_center[0]])
+        palette_map_img[img_id] = np.vstack([palette_map[1][img_id], palette_map[0][img_id]])
+
+        # print(img_id, L_idx[1][img_id], L_idx[0][img_id], p_grp_sal_size)
+        if L_idx[1][img_id].size == 0 and L_idx[0][img_id].size == 0:
+            L_idx_img[img_id] = np.array([])
+        elif L_idx[1][img_id].size == 0:
+            L_idx_img[img_id] = L_idx[0][img_id]+p_grp_sal_size
+        elif L_idx[0][img_id].size == 0:
+            L_idx_img[img_id] = L_idx[1][img_id]
+        else:
+            L_idx_img[img_id] = np.vstack([L_idx[1][img_id], L_idx[0][img_id]+p_grp_sal_size])
+        # print(L_idx_img[img_id])
+        
+    M = np.vstack([M[1], M[0]])
+    return center_img, M, palette_map_img, L_idx_img, p_src_sal_size, p_grp_sal_size
+
+
+
+
+# recolor multiple images for color consistency
+def recolor_group_images(inp_images, 
+                         num_center_grp, num_center_sal, num_center_nonsal,
+                         if_wb, 
+                         if_sal, recolor_nonsal_only, recolor_sal_only,
+                         if_cn, naming_thres, 
+                         save_dir='./results/testing'):
+
+    mode = 2
+    bin_size = 16
+
+    num_img = len(inp_images)
+    images = [None for i in range(num_img)]
+    recolored_images = [None for i in range(num_img)]
+    img_p_src = [None for i in range(num_img)]
+    img_p_tgt = [None for i in range(num_img)]
+
+    links = []
+
+    for img_id, image in enumerate(inp_images):
+        img_name = os.path.basename(image.name)
+        print('processing image {}: {}...'.format(img_id, img_name))
+        images[img_id] = BaseImage(image)
+
+    if if_sal:
+        palette_src, palette_grp, palette_tgt, links, src_sal_size, grp_sal_size = solve_grp_palette_wsal(images, mode, bin_size,
+                                                                                    lightness=70., eta=1e10, gamma=0, iteration=10,
+                                                                                    if_wb=if_wb, wb_thres=30, 
+                                                                                    if_saliency=True, sal_thres=0.5, valid_class=[0,1], 
+                                                                                    sal_center=num_center_sal, nonsal_center=num_center_nonsal, 
+                                                                                    recolor_nonsal_only=recolor_nonsal_only, recolor_sal_only=recolor_sal_only,
+                                                                                    if_cn=if_cn, naming_thres=float(naming_thres))
+    else:
+        src_sal_size = [0 for i in range(num_img)]
+        grp_sal_size = 0
+        palette_src, palette_grp, palette_tgt, links = solve_grp_palette(images, mode, bin_size,
+                                                                         if_wb=if_wb, wb_thres=30, num_center=num_center_grp,
+                                                                         lightness=70., eta=1e10, gamma=0, iteration=10,
+                                                                         if_cn=if_cn, naming_thres=float(naming_thres))
+    
+    for img_id, image in enumerate(images):
+        # print(palette_grp[img_id].shape, palette_tgt[img_id].shape)
+        if if_wb:
+            img_wb = image.get_wb_image()
+            img_lab = rgb2lab(img_wb)
+        else:
+            img_lab = image.get_lab_image()
+        img_rgb_out, _ = lab_transfer(img_lab, palette_src[img_id], palette_tgt[img_id], mask=None, mode=2)
+        img_rgb_out = np.round(img_rgb_out*255).astype(np.uint8)
+
+        out_img_path = os.path.join(save_dir, 'recolor_'+ image.filename)
+        img_bgr_out = cv2.cvtColor(img_rgb_out, cv2.COLOR_RGB2BGR)
+        cv2.imwrite(out_img_path, img_bgr_out)
+
+        recolored_images[img_id] = img_rgb_out
+
+        # self.wb_images[img_id]        = image.get_wb_image()
+        # self.saliency_images[img_id]  = image.get_saliency()
+
+        link = (links[img_id]-1).flatten()
+        link = link.tolist()
+        link = [None if item == -1 else item for item in link]
+        links.append(link)
+
+        img_p_grp = [visualize_palette(palette_grp, patch_size=20)]
+
+        # print(palette_grp, palette_src, palette_tgt)
+
+        for img_id in range(num_img):
+            img_p_src[img_id] = visualize_palette(palette_src[img_id], patch_size=20)
+            img_p_tgt[img_id] = visualize_palette(palette_tgt[img_id], patch_size=20)
+
+    return recolored_images, img_p_src, img_p_tgt, img_p_grp
+
+
+
+
+
+

recolor.py

@@ -0,0 +1,135 @@
+import numpy as np
+from skimage.color import rgb2lab, lab2rgb
+
+
+def rgb_transfer(Iin, C_src, C_tgt, mask=None):
+
+    Iin = np.array(Iin/255.).astype(np.float32)
+
+    C_src = lab2rgb(np.expand_dims(C_src, axis=0))
+    C_tgt = lab2rgb(np.expand_dims(C_tgt, axis=0))
+
+    C_src = np.squeeze(C_src, axis=0)
+    C_tgt = np.squeeze(C_tgt, axis=0)
+
+    if mask is None:
+        mask = np.ones_like(Iin[:,:,0])
+
+    m, n, b = Iin.shape
+
+    Iin = np.reshape(Iin, (m*n, b))
+    Iout = Iin.copy()
+    mask = mask.flatten()
+    
+    Iout = ab_transfer(Iin, C_src, C_tgt, mask=mask)
+    Iout = np.reshape(Iout, (m,n,b))
+
+    return Iout
+
+
+
+def lab_transfer(Iin, C_src, C_tgt, mask=None, mode=2):
+    # Convert RGB to Lab
+    # print(C_src)
+    # print(C_tgt)
+    if mask is None:
+        mask = np.ones_like(Iin[:,:,0])
+
+    Pout = C_tgt.copy()
+    m, n, b = Iin.shape
+
+    # Iin = rgb2lab(Iin)
+    Iin = np.reshape(Iin, (m*n, b))
+    Iout = Iin.copy()
+    mask = mask.flatten()
+
+    # C_src = rgb2lab(np.expand_dims(C_src, axis=0))
+    # C_tgt = rgb2lab(np.expand_dims(C_tgt, axis=0))
+
+    # C_src = np.squeeze(C_src, axis=0)
+    # C_tgt = np.squeeze(C_tgt, axis=0)
+    
+    if mode == 2:
+        Iout[:, 1:] = ab_transfer(Iin[:, 1:], C_src[:, 1:], C_tgt[:, 1:], mask=mask)
+    else:
+        Iout[:, 0:] = ab_transfer(Iin[:, 0:], C_src[:, 0:], C_tgt[:, 0:], mask=mask)
+
+    # Convert Lab to RGB
+    Iout = np.reshape(Iout, (m,n,b))
+    Iout = lab2rgb(Iout)
+
+    return Iout, Pout
+
+
+
+def ab_transfer(I_src, C_src, C_tgt, mask=None):
+    if mask is None:
+        mask = np.ones_like(I_src[:,0])
+
+    I_tgt = np.zeros_like(I_src)
+    [m, b] = I_src.shape
+
+    # remove close color
+    k = np.size(C_src, 0)
+    eps = 0.0001
+    W = np.zeros((m, k))
+    for i in range(k):
+        D = np.zeros(m)
+        for j in range(b):
+            D = D + (I_src[:, j] - C_src[i,j])**2
+        W[:, i] = 1./(D + eps)
+
+    # print(k,b)
+
+    sumW= np.sum(W, 1)
+    for j in range(k):
+        W[:, j]= W[:, j] / sumW
+
+    for i in range(k):
+        for j in range(b):
+            I_tgt[:, j] = I_tgt[:, j] + W[:, i] * (I_src[:, j] + C_tgt[i, j] - C_src[i, j])
+    
+    idx = np.argwhere(mask == 0)
+    
+    I_tgt[idx, :] = I_src[idx, :]
+
+    return I_tgt
+
+
+
+
+def lab_transfer_cls(Iin, C_src, C_tgt, mask=None, valid_class=None):
+    # Convert RGB to Lab
+    if mask is None:
+        mask = np.ones_like(Iin[:,:,0])
+    
+    Pout = C_tgt.copy()
+    m, n, b = Iin.shape
+
+    Iin = rgb2lab(Iin)
+    Iin = np.reshape(Iin, (m*n, b))
+    Iout = Iin.copy()
+    Iout_cls = np.zeros_like(Iin)
+
+    mask = mask.flatten()
+    mask_bin = np.zeros_like(mask)
+
+    # C_src = rgb2lab(np.expand_dims(C_src, axis=0))
+    # C_tgt = rgb2lab(np.expand_dims(C_tgt, axis=0))
+
+    # C_src = np.squeeze(C_src, axis=0)
+    # C_tgt = np.squeeze(C_tgt, axis=0)
+    for id_cls, cls in enumerate(valid_class):
+        if C_src[id_cls].size == 0:
+            continue
+        mask_bin[mask==cls] = 1
+        Iout_cls[:, 1:] = ab_transfer(Iin[:, 1:], C_src[id_cls][:, 1:], C_tgt[id_cls][:, 1:], mask=mask_bin)
+        idx = np.argwhere(mask_bin == 1)
+        Iout[idx, 1:] = Iout_cls[idx, 1:].copy()
+    
+    # Convert Lab to RGB
+    Iout = np.reshape(np.round(Iout), (m,n,b))
+    Iout = lab2rgb(Iout)
+
+    return Iout, Pout
+

requirements.txt

@@ -0,0 +1,9 @@
+numpy
+opencv-python
+torch
+scikit-learn
+scikit-image
+Pillow
+SciPy
+networkx
+libsvm

saliency/LDF/dataset.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+#coding=utf-8
+
+import os
+import cv2
+import torch
+import numpy as np
+from torch.utils.data import Dataset
+
+########################### Data Augmentation ###########################
+class Normalize(object):
+    def __init__(self, mean, std):
+        self.mean = mean 
+        self.std  = std
+    
+    def __call__(self, image, mask=None, body=None, detail=None):
+        image = (image - self.mean)/self.std
+        if mask is None:
+            return image
+        return image, mask/255, body/255, detail/255
+
+class RandomCrop(object):
+    def __call__(self, image, mask=None, body=None, detail=None):
+        H,W,_   = image.shape
+        randw   = np.random.randint(W/8)
+        randh   = np.random.randint(H/8)
+        offseth = 0 if randh == 0 else np.random.randint(randh)
+        offsetw = 0 if randw == 0 else np.random.randint(randw)
+        p0, p1, p2, p3 = offseth, H+offseth-randh, offsetw, W+offsetw-randw
+        if mask is None:
+            return image[p0:p1,p2:p3, :]
+        return image[p0:p1,p2:p3, :], mask[p0:p1,p2:p3], body[p0:p1,p2:p3], detail[p0:p1,p2:p3]
+
+class RandomFlip(object):
+    def __call__(self, image, mask=None, body=None, detail=None):
+        if np.random.randint(2)==0:
+            if mask is None:
+                return image[:,::-1,:].copy()
+            return image[:,::-1,:].copy(), mask[:, ::-1].copy(), body[:, ::-1].copy(), detail[:, ::-1].copy()
+        else:
+            if mask is None:
+                return image
+            return image, mask, body, detail
+
+class Resize(object):
+    def __init__(self, H, W):
+        self.H = H
+        self.W = W
+
+    def __call__(self, image, mask=None, body=None, detail=None):
+        image = cv2.resize(image, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR)
+        if mask is None:
+            return image
+        mask  = cv2.resize( mask, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR)
+        body  = cv2.resize( body, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR)
+        detail= cv2.resize( detail, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR)
+        return image, mask, body, detail
+
+class ToTensor(object):
+    def __call__(self, image, mask=None, body=None, detail=None):
+        image = torch.from_numpy(image)
+        image = image.permute(2, 0, 1)
+        if mask is None:
+            return image
+        mask  = torch.from_numpy(mask)
+        body  = torch.from_numpy(body)
+        detail= torch.from_numpy(detail)
+        return image, mask, body, detail
+
+
+########################### Config File ###########################
+class Config(object):
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.mean   = np.array([[[124.55, 118.90, 102.94]]])
+        self.std    = np.array([[[ 56.77,  55.97,  57.50]]])
+        # print('\nParameters...')
+        # for k, v in self.kwargs.items():
+        #     print('%-10s: %s'%(k, v))
+
+    def __getattr__(self, name):
+        if name in self.kwargs:
+            return self.kwargs[name]
+        else:
+            return None
+
+
+########################### Dataset Class ###########################
+class Data(Dataset):
+    def __init__(self, cfg):
+        self.cfg        = cfg
+        self.normalize  = Normalize(mean=cfg.mean, std=cfg.std)
+        self.randomcrop = RandomCrop()
+        self.randomflip = RandomFlip()
+        self.resize     = Resize(352, 352)
+        self.totensor   = ToTensor()
+
+        with open(cfg.datapath+'/'+cfg.mode+'.txt', 'r') as lines:
+            self.samples = []
+            for line in lines:
+                self.samples.append(line.strip())
+
+    def __getitem__(self, idx):
+        name  = self.samples[idx]
+        image = cv2.imread(self.cfg.datapath+'/image/'+name+'.jpg')[:,:,::-1].astype(np.float32)
+
+        if self.cfg.mode=='train':
+            mask  = cv2.imread(self.cfg.datapath+'/mask/' +name+'.png', 0).astype(np.float32)
+            body  = cv2.imread(self.cfg.datapath+'/body/' +name+'.png', 0).astype(np.float32)
+            detail= cv2.imread(self.cfg.datapath+'/detail/' +name+'.png', 0).astype(np.float32)
+            image, mask, body, detail = self.normalize(image, mask, body, detail)
+            image, mask, body, detail = self.randomcrop(image, mask, body, detail)
+            image, mask, body, detail = self.randomflip(image, mask, body, detail)
+            return image, mask, body, detail
+        else:
+            shape = image.shape[:2]
+            image = self.normalize(image)
+            image = self.resize(image)
+            image = self.totensor(image)
+            return image, shape, name
+
+    def __len__(self):
+        return len(self.samples)
+    
+    def collate(self, batch):
+        size = [224, 256, 288, 320, 352][np.random.randint(0, 5)]
+        image, mask, body, detail = [list(item) for item in zip(*batch)]
+        for i in range(len(batch)):
+            image[i] = cv2.resize(image[i], dsize=(size, size), interpolation=cv2.INTER_LINEAR)
+            mask[i]  = cv2.resize(mask[i],  dsize=(size, size), interpolation=cv2.INTER_LINEAR)
+            body[i]  = cv2.resize(body[i],  dsize=(size, size), interpolation=cv2.INTER_LINEAR)
+            detail[i]= cv2.resize(detail[i],  dsize=(size, size), interpolation=cv2.INTER_LINEAR)
+        image  = torch.from_numpy(np.stack(image, axis=0)).permute(0,3,1,2)
+        mask   = torch.from_numpy(np.stack(mask, axis=0)).unsqueeze(1)
+        body   = torch.from_numpy(np.stack(body, axis=0)).unsqueeze(1)
+        detail = torch.from_numpy(np.stack(detail, axis=0)).unsqueeze(1)
+        return image, mask, body, detail

saliency/LDF/infer.py

@@ -0,0 +1,40 @@
+
+import cv2
+import os
+import sys
+import numpy as np
+sys.dont_write_bytecode = True
+
+import torch
+from torch.utils.data import DataLoader
+
+import saliency.LDF.dataset as dataset
+from saliency.LDF.net import LDF
+
+
+## Implementation of saliency detection with LDF model.
+class Saliency_LDF:
+    def __init__(self, pretrained_model='./saliency/LDF/model-40'):
+        self.cfg = dataset.Config(snapshot=pretrained_model, mode='test')
+        self.normalize  = dataset.Normalize(mean=self.cfg.mean, std=self.cfg.std)
+        self.resize     = dataset.Resize(352, 352)
+        self.totensor   = dataset.ToTensor()
+        ## network
+        self.net = LDF(self.cfg)
+        self.net.train(False)
+        self.net.cuda()
+
+    def inference(self, img_rgb):
+        shape = img_rgb.shape[:2]
+        image = self.normalize(img_rgb)
+        image = self.resize(image)
+        image = self.totensor(image)
+
+        with torch.no_grad():
+            image = image.unsqueeze(0)
+            image = image.cuda().float()
+            outb1, outd1, out1, outb2, outd2, out2 = self.net(image, shape)
+            out  = out2
+            pred = torch.sigmoid(out[0,0]).cpu().numpy()  #[0,1]
+
+        return pred

saliency/LDF/model-40

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972a77654afdad80f99ce29968e8c02d5f14cad0aa52e21b46b1c47e1468d68e
+size 100920708

saliency/LDF/net.py

@@ -0,0 +1,216 @@
+#!/usr/bin/python3
+#coding=utf-8
+
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+def weight_init(module):
+    for n, m in module.named_children():
+        print('initialize: '+n)
+        if isinstance(m, nn.Conv2d):
+            nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')
+            if m.bias is not None:
+                nn.init.zeros_(m.bias)
+        elif isinstance(m, (nn.BatchNorm2d, nn.InstanceNorm2d)):
+            nn.init.ones_(m.weight)
+            if m.bias is not None:
+                nn.init.zeros_(m.bias)
+        elif isinstance(m, nn.Linear):
+            nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')
+            if m.bias is not None:
+                nn.init.zeros_(m.bias)
+        elif isinstance(m, nn.Sequential):
+            weight_init(m)
+        elif isinstance(m, nn.ReLU):
+            pass
+        else:
+            m.initialize()
+
+class Bottleneck(nn.Module):
+    def __init__(self, inplanes, planes, stride=1, downsample=None, dilation=1):
+        super(Bottleneck, self).__init__()
+        self.conv1      = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
+        self.bn1        = nn.BatchNorm2d(planes)
+        self.conv2      = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=(3*dilation-1)//2, bias=False, dilation=dilation)
+        self.bn2        = nn.BatchNorm2d(planes)
+        self.conv3      = nn.Conv2d(planes, planes*4, kernel_size=1, bias=False)
+        self.bn3        = nn.BatchNorm2d(planes*4)
+        self.downsample = downsample
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)), inplace=True)
+        out = F.relu(self.bn2(self.conv2(out)), inplace=True)
+        out = self.bn3(self.conv3(out))
+        if self.downsample is not None:
+            x = self.downsample(x)
+        return F.relu(out+x, inplace=True)
+
+class ResNet(nn.Module):
+    def __init__(self):
+        super(ResNet, self).__init__()
+        # self.cfg      = cfg
+        self.inplanes = 64
+        self.conv1    = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
+        self.bn1      = nn.BatchNorm2d(64)
+        self.layer1   = self.make_layer( 64, 3, stride=1, dilation=1)
+        self.layer2   = self.make_layer(128, 4, stride=2, dilation=1)
+        self.layer3   = self.make_layer(256, 6, stride=2, dilation=1)
+        self.layer4   = self.make_layer(512, 3, stride=2, dilation=1)
+        # if self.training:
+        #     self.initialize()
+
+    def make_layer(self, planes, blocks, stride, dilation):
+        downsample    = nn.Sequential(nn.Conv2d(self.inplanes, planes*4, kernel_size=1, stride=stride, bias=False), nn.BatchNorm2d(planes*4))
+        layers        = [Bottleneck(self.inplanes, planes, stride, downsample, dilation=dilation)]
+        self.inplanes = planes*4
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self.inplanes, planes, dilation=dilation))
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        out1 = F.relu(self.bn1(self.conv1(x)), inplace=True)
+        out1 = F.max_pool2d(out1, kernel_size=3, stride=2, padding=1)
+        out2 = self.layer1(out1)
+        out3 = self.layer2(out2)
+        out4 = self.layer3(out3)
+        out5 = self.layer4(out4)
+        return out1, out2, out3, out4, out5
+
+    def initialize(self):
+        self.load_state_dict(torch.load('../res/resnet50-19c8e357.pth'), strict=False)
+
+class Decoder(nn.Module):
+    def __init__(self):
+        super(Decoder, self).__init__()
+        self.conv0 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn0   = nn.BatchNorm2d(64)
+        self.conv1 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn1   = nn.BatchNorm2d(64)
+        self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn2   = nn.BatchNorm2d(64)
+        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn3   = nn.BatchNorm2d(64)
+
+    def forward(self, input1, input2=[0,0,0,0]):
+        out0 = F.relu(self.bn0(self.conv0(input1[0]+input2[0])), inplace=True)
+        out0 = F.interpolate(out0, size=input1[1].size()[2:], mode='bilinear')
+        out1 = F.relu(self.bn1(self.conv1(input1[1]+input2[1]+out0)), inplace=True)
+        out1 = F.interpolate(out1, size=input1[2].size()[2:], mode='bilinear')
+        out2 = F.relu(self.bn2(self.conv2(input1[2]+input2[2]+out1)), inplace=True)
+        out2 = F.interpolate(out2, size=input1[3].size()[2:], mode='bilinear')
+        out3 = F.relu(self.bn3(self.conv3(input1[3]+input2[3]+out2)), inplace=True)
+        return out3
+    
+    def initialize(self):
+        weight_init(self)
+
+
+class Encoder(nn.Module):
+    def __init__(self):
+        super(Encoder, self).__init__()
+        self.conv1 = nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1)
+        self.bn1   = nn.BatchNorm2d(64)
+        self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn2   = nn.BatchNorm2d(64)
+        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn3   = nn.BatchNorm2d(64)
+        self.conv4 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn4   = nn.BatchNorm2d(64)
+
+        self.conv1b = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn1b   = nn.BatchNorm2d(64)
+        self.conv2b = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn2b   = nn.BatchNorm2d(64)
+        self.conv3b = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn3b   = nn.BatchNorm2d(64)
+        self.conv4b = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn4b   = nn.BatchNorm2d(64)
+
+        self.conv1d = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn1d   = nn.BatchNorm2d(64)
+        self.conv2d = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn2d   = nn.BatchNorm2d(64)
+        self.conv3d = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn3d   = nn.BatchNorm2d(64)
+        self.conv4d = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
+        self.bn4d   = nn.BatchNorm2d(64)
+
+    def forward(self, out1):
+        out1 = F.relu(self.bn1(self.conv1(out1)), inplace=True)
+        out2 = F.max_pool2d(out1, kernel_size=2, stride=2)
+        out2 = F.relu(self.bn2(self.conv2(out2)), inplace=True)
+        out3 = F.max_pool2d(out2, kernel_size=2, stride=2)
+        out3 = F.relu(self.bn3(self.conv3(out3)), inplace=True)
+        out4 = F.max_pool2d(out3, kernel_size=2, stride=2)
+        out4 = F.relu(self.bn4(self.conv4(out4)), inplace=True)
+
+        out1b = F.relu(self.bn1b(self.conv1b(out1)), inplace=True)
+        out2b = F.relu(self.bn2b(self.conv2b(out2)), inplace=True)
+        out3b = F.relu(self.bn3b(self.conv3b(out3)), inplace=True)
+        out4b = F.relu(self.bn4b(self.conv4b(out4)), inplace=True)
+
+        out1d = F.relu(self.bn1d(self.conv1d(out1)), inplace=True)
+        out2d = F.relu(self.bn2d(self.conv2d(out2)), inplace=True)
+        out3d = F.relu(self.bn3d(self.conv3d(out3)), inplace=True)
+        out4d = F.relu(self.bn4d(self.conv4d(out4)), inplace=True)
+        return (out4b, out3b, out2b, out1b), (out4d, out3d, out2d, out1d)
+
+    def initialize(self):
+        weight_init(self)
+
+
+class LDF(nn.Module):
+    def __init__(self, cfg):
+        super(LDF, self).__init__()
+        self.cfg      = cfg
+        self.bkbone   = ResNet()
+        self.conv5b   = nn.Sequential(nn.Conv2d(2048, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv4b   = nn.Sequential(nn.Conv2d(1024, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv3b   = nn.Sequential(nn.Conv2d( 512, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv2b   = nn.Sequential(nn.Conv2d( 256, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        
+        self.conv5d   = nn.Sequential(nn.Conv2d(2048, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv4d   = nn.Sequential(nn.Conv2d(1024, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv3d   = nn.Sequential(nn.Conv2d( 512, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+        self.conv2d   = nn.Sequential(nn.Conv2d( 256, 64, kernel_size=1), nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True))
+
+        self.encoder  = Encoder()
+        self.decoderb = Decoder()
+        self.decoderd = Decoder()
+        self.linearb  = nn.Conv2d(64, 1, kernel_size=3, padding=1)
+        self.lineard  = nn.Conv2d(64, 1, kernel_size=3, padding=1)
+        self.linear   = nn.Sequential(nn.Conv2d(128, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True), nn.Conv2d(64, 1, kernel_size=3, padding=1))
+        self.initialize()
+
+    def forward(self, x, shape=None):
+        out1, out2, out3, out4, out5 = self.bkbone(x)
+        out2b, out3b, out4b, out5b   = self.conv2b(out2), self.conv3b(out3), self.conv4b(out4), self.conv5b(out5)
+        out2d, out3d, out4d, out5d   = self.conv2d(out2), self.conv3d(out3), self.conv4d(out4), self.conv5d(out5)
+
+        outb1 = self.decoderb([out5b, out4b, out3b, out2b])
+        outd1 = self.decoderd([out5d, out4d, out3d, out2d])
+        out1  = torch.cat([outb1, outd1], dim=1)
+        outb2, outd2 = self.encoder(out1)
+        outb2 = self.decoderb([out5b, out4b, out3b, out2b], outb2)
+        outd2 = self.decoderd([out5d, out4d, out3d, out2d], outd2)
+        out2  = torch.cat([outb2, outd2], dim=1)
+
+        if shape is None:
+            shape = x.size()[2:]
+        out1  = F.interpolate(self.linear(out1),   size=shape, mode='bilinear')
+        outb1 = F.interpolate(self.linearb(outb1), size=shape, mode='bilinear')
+        outd1 = F.interpolate(self.lineard(outd1), size=shape, mode='bilinear')
+
+        out2  = F.interpolate(self.linear(out2),   size=shape, mode='bilinear')
+        outb2 = F.interpolate(self.linearb(outb2), size=shape, mode='bilinear')
+        outd2 = F.interpolate(self.lineard(outd2), size=shape, mode='bilinear')
+        return outb1, outd1, out1, outb2, outd2, out2
+
+    def initialize(self):
+        if self.cfg.snapshot:
+            self.load_state_dict(torch.load(self.cfg.snapshot))
+        else:
+            weight_init(self)

saliency/fast_saliency.py

@@ -0,0 +1,590 @@
+import math
+import cv2
+import numpy as np
+import scipy.spatial.distance
+import scipy.signal
+import skimage
+import skimage.io
+import time
+from skimage.segmentation import slic
+from skimage.util import img_as_float
+import networkx as nx
+#import matplotlib.pyplot as plt
+
+
+def S(x1,x2,geodesic,sigma_clr=10):
+	return math.exp(-pow(geodesic[x1,x2],2)/(2*sigma_clr*sigma_clr))
+
+def compute_saliency_cost(smoothness,w_bg,wCtr):
+	n = len(w_bg)
+	A = np.zeros((n,n))
+	b = np.zeros((n))
+
+	for x in range(0,n):
+		A[x,x] = 2 * w_bg[x] + 2 * (wCtr[x])
+		b[x] = 2 * wCtr[x]
+		for y in range(0,n):
+			A[x,x] += 2 * smoothness[x,y]
+			A[x,y] -= 2 * smoothness[x,y]
+	
+	x = np.linalg.solve(A, b)
+
+	return x
+
+def path_length(path,G):
+	dist = 0.0
+	for i in range(1,len(path)):
+		dist += G[path[i - 1]][path[i]]['weight']
+	return dist
+
+def make_graph(grid):
+	# get unique labels
+	vertices = np.unique(grid)
+ 
+	# map unique labels to [1,...,num_labels]
+	reverse_dict = dict(zip(vertices,np.arange(len(vertices))))
+	grid = np.array([reverse_dict[x] for x in grid.flat]).reshape(grid.shape)
+   
+	# create edges
+	down = np.c_[grid[:-1, :].ravel(), grid[1:, :].ravel()]
+	right = np.c_[grid[:, :-1].ravel(), grid[:, 1:].ravel()]
+	all_edges = np.vstack([right, down])
+	all_edges = all_edges[all_edges[:, 0] != all_edges[:, 1], :]
+	all_edges = np.sort(all_edges,axis=1)
+	num_vertices = len(vertices)
+	edge_hash = all_edges[:,0] + num_vertices * all_edges[:, 1]
+	# find unique connections
+	edges = np.unique(edge_hash)
+	# undo hashing
+	edges = [[vertices[x%num_vertices],
+			  vertices[x//num_vertices]] for x in edges] 
+ 
+	return vertices, edges
+	
+
+# Saliency map calculation based on:
+# Wangjiang Zhu, Shuang Liang, Yichen Wei and Jian Sun, 
+# Saliency Optimization from Robust Background Detection, (CVPR), 2014
+
+# based on the asumption that saliency region has smaller 'boundary connectivity'
+# object regions (salient regions) are much less connected to image boundaries than background ones
+# superpixel
+
+def get_saliency_rbd(img):
+
+	img_lab = img_as_float(skimage.color.rgb2lab(img))
+
+	img_rgb = img_as_float(img)
+
+	img_gray = img_as_float(skimage.color.rgb2gray(img))
+
+	segments_slic = slic(img_rgb, n_segments=250, compactness=10, sigma=1, enforce_connectivity=False)
+
+	# num_segments = len(np.unique(segments_slic))
+
+	nrows, ncols = segments_slic.shape
+	max_dist = math.sqrt(nrows*nrows + ncols*ncols)
+
+	grid = segments_slic
+
+	(vertices,edges) = make_graph(grid)
+
+	gridx, gridy = np.mgrid[:grid.shape[0], :grid.shape[1]]
+
+	centers = dict()
+	colors = dict()
+	# distances = dict()
+	boundary = dict()
+
+	for v in vertices:
+		centers[v] = [gridy[grid == v].mean(), gridx[grid == v].mean()]
+		colors[v] = np.mean(img_lab[grid==v],axis=0)
+
+		x_pix = gridx[grid == v]
+		y_pix = gridy[grid == v]
+
+		if np.any(x_pix == 0) or np.any(y_pix == 0) or np.any(x_pix == nrows - 1) or np.any(y_pix == ncols - 1):
+			boundary[v] = 1
+		else:
+			boundary[v] = 0
+
+	G = nx.Graph()
+
+	#buid the graph
+	for edge in edges:
+		pt1 = edge[0]
+		pt2 = edge[1]
+		color_distance = scipy.spatial.distance.euclidean(colors[pt1],colors[pt2])
+		G.add_edge(pt1, pt2, weight=color_distance )
+
+	#add a new edge in graph if edges are both on boundary
+	for v1 in vertices:
+		if boundary[v1] == 1:
+			for v2 in vertices:
+				if boundary[v2] == 1:
+					color_distance = scipy.spatial.distance.euclidean(colors[v1],colors[v2])
+					G.add_edge(v1,v2,weight=color_distance)
+
+	geodesic = np.zeros((len(vertices),len(vertices)),dtype=float)
+	spatial = np.zeros((len(vertices),len(vertices)),dtype=float)
+	smoothness = np.zeros((len(vertices),len(vertices)),dtype=float)
+	adjacency = np.zeros((len(vertices),len(vertices)),dtype=float)
+
+	sigma_clr = 10.0
+	sigma_bndcon = 1.0
+	sigma_spa = 0.25
+	mu = 0.1
+
+	all_shortest_paths_color = nx.shortest_path(G,source=None,target=None,weight='weight')
+
+	for v1 in vertices:
+		for v2 in vertices:
+			if v1 == v2:
+				geodesic[v1,v2] = 0
+				spatial[v1,v2] = 0
+				smoothness[v1,v2] = 0
+			else:
+				geodesic[v1,v2] = path_length(all_shortest_paths_color[v1][v2],G)
+				spatial[v1,v2] = scipy.spatial.distance.euclidean(centers[v1],centers[v2]) / max_dist
+				smoothness[v1,v2] = math.exp( - (geodesic[v1,v2] * geodesic[v1,v2])/(2.0*sigma_clr*sigma_clr)) + mu 
+
+	for edge in edges:
+		pt1 = edge[0]
+		pt2 = edge[1]
+		adjacency[pt1,pt2] = 1
+		adjacency[pt2,pt1] = 1
+
+	for v1 in vertices:
+		for v2 in vertices:
+			smoothness[v1,v2] = adjacency[v1,v2] * smoothness[v1,v2]
+
+	area = dict()
+	len_bnd = dict()
+	bnd_con = dict()
+	w_bg = dict()
+	ctr = dict()
+	wCtr = dict()
+
+	for v1 in vertices:
+		area[v1] = 0
+		len_bnd[v1] = 0
+		ctr[v1] = 0
+		for v2 in vertices:
+			d_app = geodesic[v1,v2]
+			d_spa = spatial[v1,v2]
+			w_spa = math.exp(- ((d_spa)*(d_spa))/(2.0*sigma_spa*sigma_spa))
+			area_i = S(v1,v2,geodesic)
+			area[v1] += area_i
+			len_bnd[v1] += area_i * boundary[v2]
+			ctr[v1] += d_app * w_spa
+		bnd_con[v1] = len_bnd[v1] / math.sqrt(area[v1])
+		w_bg[v1] = 1.0 - math.exp(- (bnd_con[v1]*bnd_con[v1])/(2*sigma_bndcon*sigma_bndcon))
+
+	for v1 in vertices:
+		wCtr[v1] = 0
+		for v2 in vertices:
+			d_app = geodesic[v1,v2]
+			d_spa = spatial[v1,v2]
+			w_spa = math.exp(- (d_spa*d_spa)/(2.0*sigma_spa*sigma_spa))
+			wCtr[v1] += d_app * w_spa *  w_bg[v2]
+
+	# normalise value for wCtr
+
+	min_value = min(wCtr.values())
+	max_value = max(wCtr.values())
+
+	minVal = [key for key, value in wCtr.items() if value == min_value]
+	maxVal = [key for key, value in wCtr.items() if value == max_value]
+
+	for v in vertices:
+		wCtr[v] = (wCtr[v] - min_value)/(max_value - min_value)
+
+	img_disp1 = img_gray.copy()
+	# img_disp2 = img_gray.copy()
+
+	x = compute_saliency_cost(smoothness,w_bg,wCtr)
+
+	for v in vertices:
+		img_disp1[grid == v] = x[v]
+
+	img_disp2 = img_disp1.copy()
+	sal = np.zeros((img_disp1.shape[0],img_disp1.shape[1],3))
+
+	sal = img_disp2
+	sal_max = np.max(sal)
+	sal_min = np.min(sal)
+	sal = ((sal - sal_min) / (sal_max - sal_min))
+
+	return sal
+
+
+
+# Saliency map calculation based on:
+# R. Achanta, S. Hemami, F. Estrada and S. Süsstrunk, 
+# Frequency-tuned Salient Region Detection, (CVPR 2009), pp. 1597 - 1604, 2009
+# a frequency-tuned approach to estimate center-surround contrast using color and luminance features
+# combine several different filters to remove unwanted high-frequency components 
+
+def get_saliency_ft(img):
+
+	img_rgb = img_as_float(img)
+
+	img_lab = skimage.color.rgb2lab(img_rgb) 
+
+	mean_val = np.mean(img_rgb,axis=(0,1))
+
+	kernel_h = (1.0/16.0) * np.array([[1,4,6,4,1]])
+	kernel_w = kernel_h.transpose()
+
+	blurred_l = scipy.signal.convolve2d(img_lab[:,:,0],kernel_h,mode='same')
+	blurred_a = scipy.signal.convolve2d(img_lab[:,:,1],kernel_h,mode='same')
+	blurred_b = scipy.signal.convolve2d(img_lab[:,:,2],kernel_h,mode='same')
+
+	blurred_l = scipy.signal.convolve2d(blurred_l,kernel_w,mode='same')
+	blurred_a = scipy.signal.convolve2d(blurred_a,kernel_w,mode='same')
+	blurred_b = scipy.signal.convolve2d(blurred_b,kernel_w,mode='same')
+
+	im_blurred = np.dstack([blurred_l,blurred_a,blurred_b])
+
+	sal = np.linalg.norm(mean_val - im_blurred,axis = 2)
+	sal_max = np.max(sal)
+	sal_min = np.min(sal)
+	sal = ((sal - sal_min) / (sal_max - sal_min))
+	return sal
+
+
+
+def raster_scan(img,L,U,D):
+	n_rows = len(img)
+	n_cols = len(img[0])
+
+	for x in range(1, n_rows - 1):
+		for y in range(1, n_cols - 1):
+			ix = img[x][y]
+			d = D[x][y]
+
+			u1 = U[x-1][y]
+			l1 = L[x-1][y]
+
+			u2 = U[x][y-1]
+			l2 = L[x][y-1]
+
+			b1 = max(u1,ix) - min(l1,ix)
+			b2 = max(u2,ix) - min(l2,ix)
+
+			if d <= b1 and d <= b2:
+				continue
+			elif b1 < d and b1 <= b2:
+				D[x][y] = b1
+				U[x][y] = max(u1, ix)
+				L[x][y] = min(l1, ix)
+			else:
+				D[x][y] = b2
+				U[x][y] = max(u2, ix)
+				L[x][y] = min(l2, ix)
+	return True
+
+
+def raster_scan(img, L, U, D):
+	n_rows = len(img)
+	n_cols = len(img[0])
+
+	for x in range(1, n_rows - 1):
+		for y in range(1, n_cols - 1):
+			ix = img[x][y]
+			d = D[x][y]
+
+			u1 = U[x-1][y]
+			l1 = L[x-1][y]
+
+			u2 = U[x][y-1]
+			l2 = L[x][y-1]
+
+			b1 = max(u1,ix) - min(l1,ix)
+			b2 = max(u2,ix) - min(l2,ix)
+
+			if d <= b1 and d <= b2:
+				continue
+			elif b1 < d and b1 <= b2:
+				D[x][y] = b1
+				U[x][y] = max(u1, ix)
+				L[x][y] = min(l1, ix)
+			else:
+				D[x][y] = b2
+				U[x][y] = max(u2, ix)
+				L[x][y] = min(l2, ix)
+	return True
+
+
+
+
+
+def raster_scan_inv(img,L,U,D):
+	n_rows = len(img)
+	n_cols = len(img[0])
+
+	for x in range(n_rows - 2, 1, -1):
+		for y in range(n_cols - 2, 1, -1):
+
+			ix = img[x][y]
+			d = D[x][y]
+
+			u1 = U[x+1][y]
+			l1 = L[x+1][y]
+
+			u2 = U[x][y+1]
+			l2 = L[x][y+1]
+
+			b1 = max(u1,ix) - min(l1,ix)
+			b2 = max(u2,ix) - min(l2,ix)
+
+			if d <= b1 and d <= b2:
+				continue
+			elif b1 < d and b1 <= b2:
+				D[x][y] = b1
+				U[x][y] = max(u1,ix)
+				L[x][y] = min(l1,ix)
+			else:
+				D[x][y] = b2
+				U[x][y] = max(u2,ix)
+				L[x][y] = min(l2,ix)
+	return True
+
+
+def mbd(img, num_iters):
+	if len(img.shape) != 2:
+		print('did not get 2d np array to fast mbd')
+		return None
+	if (img.shape[0] <= 3 or img.shape[1] <= 3):
+		print('image is too small')
+		return None
+
+	L = np.copy(img)
+	U = np.copy(img)
+	D = float('Inf') * np.ones(img.shape)
+	D[0,:] = 0
+	D[-1,:] = 0
+	D[:,0] = 0
+	D[:,-1] = 0
+
+	# unfortunately, iterating over numpy arrays is very slow
+	img_list = img.tolist()
+	L_list = L.tolist()
+	U_list = U.tolist()
+	D_list = D.tolist()
+
+	# start_time = time.time()
+	for x in range(0, num_iters):
+		if x%2 == 1:
+			raster_scan(img_list, L_list, U_list, D_list)
+		else:
+			raster_scan_inv(img_list, L_list, U_list, D_list)
+
+	# end_time = time.time()
+	# print('mbd function: ', end_time-start_time)
+	return np.array(D_list)
+
+
+# Saliency map calculation based on:
+# Minimum Barrier Salient Object Detection at 80 FPS
+# based on the Image Boundary Connectivity Cue, which assumes that 
+# background regions are usually connected to the image borders
+# cons: 
+# doesn't consider spatial info, distant objects are detected as the same object
+# fail when the contrast between foreground and background are small
+
+def get_saliency_mbd(img):
+
+	img_mean = np.mean(img, axis=(2))
+	# start_time = time.time()
+	sal = mbd(img_mean,3)
+	# end_time = time.time()
+	# print('mbd function: ', end_time-start_time)
+
+	# get the background map
+	# paper uses 30px for an image of size 300px, so we use 10%
+	(n_rows, n_cols, n_channels) = img.shape
+	img_size = math.sqrt(n_rows * n_cols)
+	border_thickness = int(math.floor(0.1 * img_size))
+
+	img_lab = img_as_float(skimage.color.rgb2lab(img))
+	
+	px_left = img_lab[0:border_thickness,:,:]
+	px_right = img_lab[n_rows - border_thickness-1:-1,:,:]
+
+	px_top = img_lab[:,0:border_thickness,:]
+	px_bottom = img_lab[:,n_cols - border_thickness-1:-1,:]
+	
+	px_mean_left = np.mean(px_left,axis=(0,1))
+	px_mean_right = np.mean(px_right,axis=(0,1))
+	px_mean_top = np.mean(px_top,axis=(0,1))
+	px_mean_bottom = np.mean(px_bottom,axis=(0,1))
+
+
+	px_left = px_left.reshape((n_cols*border_thickness,3))
+	px_right = px_right.reshape((n_cols*border_thickness,3))
+
+	px_top = px_top.reshape((n_rows*border_thickness,3))
+	px_bottom = px_bottom.reshape((n_rows*border_thickness,3))
+
+	cov_left = np.cov(px_left.T)
+	cov_right = np.cov(px_right.T)
+
+	cov_top = np.cov(px_top.T)
+	cov_bottom = np.cov(px_bottom.T)
+
+	cov_left = np.linalg.inv(cov_left)
+	cov_right = np.linalg.inv(cov_right)
+	
+	cov_top = np.linalg.inv(cov_top)
+	cov_bottom = np.linalg.inv(cov_bottom)
+
+
+	u_left = np.zeros(sal.shape)
+	u_right = np.zeros(sal.shape)
+	u_top = np.zeros(sal.shape)
+	u_bottom = np.zeros(sal.shape)
+
+	u_final = np.zeros(sal.shape)
+	img_lab_unrolled = img_lab.reshape(img_lab.shape[0]*img_lab.shape[1],3)
+
+	px_mean_left_2 = np.zeros((1,3))
+	px_mean_left_2[0,:] = px_mean_left
+
+	u_left = scipy.spatial.distance.cdist(img_lab_unrolled,px_mean_left_2,'mahalanobis', VI=cov_left)
+	u_left = u_left.reshape((img_lab.shape[0],img_lab.shape[1]))
+
+	px_mean_right_2 = np.zeros((1,3))
+	px_mean_right_2[0,:] = px_mean_right
+
+	u_right = scipy.spatial.distance.cdist(img_lab_unrolled,px_mean_right_2,'mahalanobis', VI=cov_right)
+	u_right = u_right.reshape((img_lab.shape[0],img_lab.shape[1]))
+
+	px_mean_top_2 = np.zeros((1,3))
+	px_mean_top_2[0,:] = px_mean_top
+
+	u_top = scipy.spatial.distance.cdist(img_lab_unrolled,px_mean_top_2,'mahalanobis', VI=cov_top)
+	u_top = u_top.reshape((img_lab.shape[0],img_lab.shape[1]))
+
+	px_mean_bottom_2 = np.zeros((1,3))
+	px_mean_bottom_2[0,:] = px_mean_bottom
+
+	u_bottom = scipy.spatial.distance.cdist(img_lab_unrolled,px_mean_bottom_2,'mahalanobis', VI=cov_bottom)
+	u_bottom = u_bottom.reshape((img_lab.shape[0],img_lab.shape[1]))
+
+	max_u_left = np.max(u_left)
+	max_u_right = np.max(u_right)
+	max_u_top = np.max(u_top)
+	max_u_bottom = np.max(u_bottom)
+
+	u_left = u_left / max_u_left
+	u_right = u_right / max_u_right
+	u_top = u_top / max_u_top
+	u_bottom = u_bottom / max_u_bottom
+
+	u_max = np.maximum(np.maximum(np.maximum(u_left,u_right),u_top),u_bottom)
+
+	u_final = (u_left + u_right + u_top + u_bottom) - u_max
+
+	u_max_final = np.max(u_final)
+	sal_max = np.max(sal)
+	sal = sal / sal_max + u_final / u_max_final
+
+	#postprocessing
+	# apply centredness map
+	sal = sal / np.max(sal)
+	
+	s = np.mean(sal)
+	alpha = 50.0
+	delta = alpha * math.sqrt(s)
+
+	xv,yv = np.meshgrid(np.arange(sal.shape[1]),np.arange(sal.shape[0]))
+	(w,h) = sal.shape
+	w2 = w/2.0
+	h2 = h/2.0
+
+	C = 1 - np.sqrt(np.power(xv - h2,2) + np.power(yv - w2,2)) / math.sqrt(np.power(w2,2) + np.power(h2,2))
+	sal = sal * C
+
+	#increase bg/fg contrast 
+	def f(x):
+		b = 10.0
+		return 1.0 / (1.0 + math.exp(-b*(x - 0.5)))
+
+	fv = np.vectorize(f)
+	sal = sal / np.max(sal)
+	sal = fv(sal)
+	return sal
+	
+
+def binarise_saliency_map(saliency_map, method='adaptive',threshold=0.5):
+
+	# check if input is a numpy array
+	if type(saliency_map).__module__ != np.__name__:
+		print('Expected numpy array')
+		return None
+
+	#check if input is 2D
+	if len(saliency_map.shape) != 2:
+		print('Saliency map must be 2D')
+		return None
+
+	if method == 'fixed':
+		return (saliency_map > threshold)
+
+	elif method == 'adaptive':
+		adaptive_threshold = 2.0 * saliency_map.mean()
+		return (saliency_map > adaptive_threshold)
+
+	elif method == 'clustering':
+		print('Not yet implemented')
+		return None
+
+	else:
+		print("Method must be one of fixed, adaptive or clustering")
+		return None
+
+
+if __name__ == '__main__':
+	# path to the image
+	filename = '../images/flower/19569518092_2db12519fd_c.jpg'
+	# filename = './images/landmark_04/12004354405_dc546d53ce_c.jpg'
+
+	img = skimage.io.imread(filename)
+
+	if len(img.shape) != 3: # got a grayscale image
+		img = skimage.color.gray2rgb(img)
+
+	# get the saliency maps using the 3 implemented methods
+	start_time = time.time()
+	rbd = get_saliency_rbd(img)
+	end_time = time.time()
+	print('rbd:', end_time-start_time)
+
+	start_time = time.time()
+	ft = get_saliency_ft(img)
+	end_time = time.time()
+	print('ft:', end_time-start_time)
+
+	start_time = time.time()
+	mbd_img = get_saliency_mbd(img)
+	end_time = time.time()
+	print('mbd:', end_time-start_time)
+
+	# often, it is desirable to have a binary saliency map
+	binary_sal = binarise_saliency_map(mbd_img, method='adaptive')
+
+	img = cv2.imread(filename)
+
+	# print(mbd.max())
+
+	cv2.imshow('img', img)
+	cv2.imshow('rbd', rbd)
+	cv2.imshow('ft', ft)
+	cv2.imshow('mbd', mbd_img)
+
+	#openCV cannot display numpy type 0, so convert to uint8 and scale
+	cv2.imshow('binary', 255 * binary_sal.astype('uint8'))
+
+
+	cv2.waitKey(0)

solve_group_palette.py

@@ -0,0 +1,240 @@
+
+import numpy as np
+from sklearn.cluster import KMeans
+from sklearn.metrics import pairwise_distances
+
+from utils import stack_list
+
+
+
+def solve_group_palette(hist_sample_all, hist_counts_all, center, density, reference, m, lightness=70, eta=0, gamma=1e10, iteration=10):
+    num_img = len(center)
+    Lout = [i for i in range(num_img)]
+
+    if num_img > 1:
+        gamma = gamma / ((num_img-1)/2)
+    lbd = gamma / 50
+
+    old_val = 0
+    init_min=np.Inf
+
+    if m == 1:
+        M = np.mean(hist_sample_all, axis=0)
+    else:
+        cinits = np.zeros((m, np.size(hist_sample_all, 1)))
+        cw = hist_counts_all
+        for i in range(m):
+            id = np.argmax(cw)
+            cinits[i,:] = hist_sample_all[id,:]
+            d2 = cinits[i,:]* np.ones((np.size(hist_sample_all, 0), 1)) - hist_sample_all
+            d2 = np.sum(np.square(d2), axis=1)
+            d2 = d2/np.max(d2)
+            cw = cw * (d2**2)
+
+        kmeans_grp = KMeans(n_clusters=m, init=cinits, n_init=1).fit(
+                    hist_sample_all, y=None, sample_weight=hist_counts_all)
+        M = kmeans_grp.cluster_centers_
+
+    if np.size(hist_sample_all, 1) == 2:
+        # print(M.shape)
+        if M.ndim == 1:
+            M = np.expand_dims(M, axis=0)
+        M = np.insert(M, 0, values=lightness, axis=1)
+        # print(M.shape)
+
+    ## choose the nearest cluster center from all the individual palettes as the inital group palette
+        
+    # center_r0 = delete_num(center, 0)
+    # density_r0 = delete_num(density, 0)
+    # center_all = center_r0[0]
+    # for i in range(len(center_r0)-1):
+    #     center_all = np.vstack([center_all, center_r0[i+1]])
+        
+    center_r0 = delete_num(center)
+    density_r0 = delete_num(density)
+    center_all = stack_list(center)
+    
+
+    if M.ndim == 1:
+        M=M.reshape(1, -1)
+
+    D = pairwise_distances(M, center_all, metric='euclidean')
+    idx = np.argmin(D, 1)
+
+    # center_all
+
+    M = center_all[idx,:]
+
+    ## solve the group palette according to the requirement (gamma and eta)
+    for t in range(iteration):
+        sum_val = 0
+        # solve for the assignment (matching)
+        for i in range(num_img):
+            # print(center[i])
+            if center[i].size != 0:
+            # if center[i] is not 0:
+                # print(center[i])
+                Lout[i], val = solve_optimal_ind_palette(center[i], density[i], M, lbd, gamma, eta, init_min)
+                sum_val = sum_val + val
+            else:
+                Lout[i] = np.array([])
+                sum_val = 0
+
+        # re-compute the group color theme (mean colors)
+        Lout_r0 = delete_num(Lout)
+        idx = detect_num(Lout)
+        reference_r0 = reference[idx]
+        M = solve_mean(center_r0, reference_r0, density_r0, Lout_r0, m, len(reference_r0), lbd)
+        print('Iteration {}, val: {}'.format(t, sum_val))
+        if np.abs(old_val - sum_val) < 10:
+            break
+        else:
+            old_val = sum_val
+
+    return M, Lout
+
+
+def delete_num(list_org):
+    list_new=[]
+    for i in list_org:
+        if i.size != 0:
+            list_new.append(i)
+    return list_new
+
+def detect_num(list_org):
+    list_idx=[]
+    for i in range(len(list_org)):
+        if list_org[i].size != 0:
+            list_idx.append(i)
+    return list_idx
+
+
+def solve_optimal_ind_palette(center, density, center_mean, lambd, gamma, eta, init_min):
+
+    n = np.size(center, 0)
+    m = np.size(center_mean, 0)
+    # print(eta)
+
+    # brute-force all possible cases
+    min_obj_func = init_min
+    D1 = pairwise_distances(center, center, metric='euclidean')
+    D2 = psub2(center, center_mean)
+    
+    dist = pairwise_distances(center, center_mean, metric='euclidean')
+    dist = np.tile(np.expand_dims(density, axis=1), (1,m)) * (dist**2)
+
+    D3 = np.insert(dist, 0, values=np.zeros(n), axis=1)
+
+    
+    num_of_pairs = (m+1)**n
+    label = np.zeros((n,1)).astype(np.int32)
+    min_label_com = label.copy()
+    label[n-1, :] = -1
+    for idx in range(num_of_pairs):
+        label[n-1] = label[n-1] + 1
+        curId = n-1
+        while label[curId] > m:
+            label[curId] = 0
+            curId = curId - 1
+            label[curId] = label[curId] + 1
+        term4 = np.sum((label==0) * np.expand_dims(density, axis=1))
+        val = eta * term4
+        if val >= min_obj_func:
+            continue
+        for i in range(n):
+            val = val + D3[i, label[i]] #the first term
+        # cut down the unsolution
+        if val >= min_obj_func:
+            continue
+        term2 = 0
+        term3 = 0
+        for ii in range(n-1):
+            for jj in range(ii+1, n):
+                term2 = term2 + D2[ii, jj, label[ii], label[jj]]
+                if label[ii] == label[jj] and label[ii] > 0:
+                    term3 = term3 + D1[ii, jj]
+
+        val = val + lambd * term2 + gamma * term3
+        if val < min_obj_func:
+            min_obj_func = val.copy()
+            min_label_com = label.copy()
+
+    return min_label_com, min_obj_func
+
+
+
+
+def solve_mean(center, reference, density, L, m, n, lambd):
+    A = np.zeros((m, m))
+    B = np.zeros((m, 3))
+    M = np.zeros((m, 3))
+
+    for i in range(n):
+        if reference[i] == 0:
+            continue
+        Pi = center[i]
+        Wi = density[i]
+        Li = L[i]
+        ni = np.size(Pi, 0)
+        # first term
+        for j in range(ni):
+            if Li[j] > 0:
+                A[Li[j]-1, Li[j]-1] = A[Li[j]-1, Li[j]-1] + Wi[j]
+                B[Li[j]-1, :] = B[Li[j]-1, :] + np.expand_dims(Wi[j] * Pi[j, :], axis=0)
+        
+        # second term
+        for j1 in range(ni-1):
+            for j2 in range(j1+1, ni):
+                if Li[j1] > 0 and Li[j2] > 0:
+                    A[Li[j1]-1, Li[j1]-1] = A[Li[j1]-1, Li[j1]-1] + lambd
+                    A[Li[j1]-1, Li[j2]-1] = A[Li[j1]-1, Li[j2]-1] - lambd
+                    A[Li[j2]-1, Li[j2]-1] = A[Li[j2]-1, Li[j2]-1] + lambd
+                    A[Li[j2]-1, Li[j1]-1] = A[Li[j2]-1, Li[j1]-1] - lambd
+                    B[Li[j1]-1, :] = B[Li[j1]-1, :] + lambd * (Pi[j1,:]-Pi[j2,:])
+                    B[Li[j2]-1, :] = B[Li[j2]-1, :] + lambd * (Pi[j2,:]-Pi[j1,:])
+
+    # solve least squares
+    # print(A)
+    M = np.dot(np.linalg.pinv(A), B)
+
+    # M[np.isnan[M]] = 0
+
+    # k-median, take the nearest from the individual palettes
+    M = choose_mediod(center, M)
+    
+    return M
+
+
+
+def concat_list(input, axis=0):
+    list_cat = input[0]
+    for i in range(len(input)-1):
+        list_cat = np.concatenate((list_cat, input[i+1]), axis=axis)
+        # list_cat = np.vstack([list_cat, input[i+1]])
+    return list_cat
+
+
+def choose_mediod(Pin, M):
+    P = concat_list(Pin)
+    D = pairwise_distances(M, P)
+    idx = np.argmin(D, axis=1)
+    M = P[idx,:]
+    return M
+
+
+
+def psub2(P, M):
+    p = np.size(P, 0)
+    m = np.size(M, 0)
+    D = np.zeros((p, p, m + 1, m + 1))
+    for i1 in range(p-1):
+        for i2 in range(i1+1, p):
+            for i3 in range(m):
+                for i4 in range(m):
+                    D[i1,i2,i3+1,i4+1] = np.sum((P[i1,:] - P[i2,:] - M[i3,:] + M[i4,:])**2)
+    return D
+
+
+
+
+

utils.py

@@ -0,0 +1,124 @@
+# import cv2
+import numpy as np
+# import matplotlib.pyplot as plt
+from skimage.color import rgb2lab, lab2rgb, rgb2hsv, hsv2rgb
+from PIL import Image
+
+
+def stack_list(x):
+    stack = []
+    for id, val in enumerate(x):
+        if np.size(val) != 0:
+            if stack == []:
+                stack = val
+            else:
+                stack = np.vstack([stack, val])
+    return stack   
+
+def rgb_to_hex(r, g, b):
+    return '#{:02x}{:02x}{:02x}'.format(r, g, b)
+
+def hex_to_rgb(hex):
+    # print(hex)
+    rgb = []
+    for i in (1, 3, 5):
+        decimal = int(hex[i:i+2], 16)
+        rgb.append(decimal)
+    return tuple(rgb)
+
+
+def image_resize(img, c_w, c_h):
+    # img : PIL Image
+    if type(img) is np.ndarray:
+        img = Image.fromarray(img)
+    h, w = img.size
+    h_factor = c_h / w
+    w_factor = c_w / h
+    # factor = h_factor
+    factor = np.minimum(h_factor, w_factor)
+    # print(w*factor, h*factor)
+    img = img.resize((np.round(h*factor).astype(np.int64), 
+                     np.round(w*factor).astype(np.int64)),
+                     Image.BILINEAR)
+    return img
+
+def get_palette(num_cls):
+    """ Returns the color map for visualizing the segmentation mask.
+    Args:
+        num_cls: Number of classes
+    Returns:
+        The color map
+    """
+    n = num_cls
+    palette = [0] * (n * 3)
+    for j in range(0, n):
+        lab = j
+        palette[j * 3 + 0] = 0
+        palette[j * 3 + 1] = 0
+        palette[j * 3 + 2] = 0
+        i = 0
+        while lab:
+            palette[j * 3 + 0] |= (((lab >> 0) & 1) << (7 - i))
+            palette[j * 3 + 1] |= (((lab >> 1) & 1) << (7 - i))
+            palette[j * 3 + 2] |= (((lab >> 2) & 1) << (7 - i))
+            i += 1
+            lab >>= 3
+    return palette
+
+
+def visualize_palette(palette_lab, patch_size=20):
+    # print(palette_lab)
+    if palette_lab is None:
+        return np.ones((patch_size, patch_size, 3)) * [1.,1.,1.]
+    palette_lab = np.expand_dims(palette_lab, axis=0)
+    # palette_lab = np.sort(palette_lab, axis=1)
+
+    # # lab transfer by lookuptable
+    # # cluster_cen_rgb = lab2rgb_lut(cluster_cen_lab)
+    palette_rgb = lab2rgb(palette_lab)
+    palette_rgb = np.squeeze(palette_rgb, axis=0)
+
+    for id in range(np.size(palette_rgb, 0)):
+        rgb = np.expand_dims(palette_rgb[id,:], axis=(0, 1))
+        if id==0:
+            img_palette = np.ones((patch_size, patch_size, 3)) * rgb
+        else:
+            img_palette = np.append(img_palette, np.ones((patch_size, patch_size, 3)) * rgb, axis=1)
+    
+    return img_palette
+
+
+def visualize_palette_rgb(palette_rgb, patch_size=20):
+    # print(palette_lab)
+    if palette_rgb == 0:
+        return np.ones((patch_size, patch_size, 3)) * [1.,1.,1.]
+
+
+    for id in range(np.size(palette_rgb, 0)):
+        rgb = np.expand_dims(palette_rgb[id,:], axis=(0, 1))
+        if id==0:
+            img_palette = np.ones((patch_size, patch_size, 3)) * rgb
+        else:
+            img_palette = np.append(img_palette, np.ones((patch_size, patch_size, 3)) * rgb, axis=1)
+    
+    return img_palette
+
+
+
+# def vis_consistency(img_rgb_all, img_rgb_out_all, label_colored_all, c_center, L_idx):
+
+
+
+def color_difference(img1, img2):
+    h, w, c = img1.shape
+    img1_lab = rgb2lab(img1)
+    img2_lab = rgb2lab(img2)
+
+    diff=img1_lab-img2_lab
+    
+    dE = np.sqrt(diff[:,:,0]**2 + diff[:,:,1]**2 + diff[:,:,2]**2)
+    # dE = np.sqrt(diff[:,:,0]**2 + diff[:,:,0]**2)
+    dE = np.sum(dE)/(h*w)
+
+    return dE
+