init
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import copy
|
||||
|
||||
__all__ = ["build_metric"]
|
||||
|
||||
from .det_metric import DetMetric, DetFCEMetric
|
||||
from .rec_metric import RecMetric, CNTMetric, CANMetric
|
||||
from .cls_metric import ClsMetric
|
||||
from .e2e_metric import E2EMetric
|
||||
from .distillation_metric import DistillationMetric
|
||||
from .table_metric import TableMetric
|
||||
from .kie_metric import KIEMetric
|
||||
from .vqa_token_ser_metric import VQASerTokenMetric
|
||||
from .vqa_token_re_metric import VQAReTokenMetric
|
||||
from .sr_metric import SRMetric
|
||||
from .ct_metric import CTMetric
|
||||
|
||||
|
||||
def build_metric(config):
|
||||
support_dict = [
|
||||
"DetMetric", "DetFCEMetric", "RecMetric", "ClsMetric", "E2EMetric",
|
||||
"DistillationMetric", "TableMetric", 'KIEMetric', 'VQASerTokenMetric',
|
||||
'VQAReTokenMetric', 'SRMetric', 'CTMetric', 'CNTMetric', 'CANMetric'
|
||||
]
|
||||
|
||||
config = copy.deepcopy(config)
|
||||
module_name = config.pop("name")
|
||||
assert module_name in support_dict, Exception(
|
||||
"metric only support {}".format(support_dict))
|
||||
module_class = eval(module_name)(**config)
|
||||
return module_class
|
||||
@@ -0,0 +1,46 @@
|
||||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class ClsMetric(object):
|
||||
def __init__(self, main_indicator='acc', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.eps = 1e-5
|
||||
self.reset()
|
||||
|
||||
def __call__(self, pred_label, *args, **kwargs):
|
||||
preds, labels = pred_label
|
||||
correct_num = 0
|
||||
all_num = 0
|
||||
for (pred, pred_conf), (target, _) in zip(preds, labels):
|
||||
if pred == target:
|
||||
correct_num += 1
|
||||
all_num += 1
|
||||
self.correct_num += correct_num
|
||||
self.all_num += all_num
|
||||
return {'acc': correct_num / (all_num + self.eps), }
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0
|
||||
}
|
||||
"""
|
||||
acc = self.correct_num / (self.all_num + self.eps)
|
||||
self.reset()
|
||||
return {'acc': acc}
|
||||
|
||||
def reset(self):
|
||||
self.correct_num = 0
|
||||
self.all_num = 0
|
||||
@@ -0,0 +1,52 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
from scipy import io
|
||||
import numpy as np
|
||||
|
||||
from ppocr.utils.e2e_metric.Deteval import combine_results, get_score_C
|
||||
|
||||
|
||||
class CTMetric(object):
|
||||
def __init__(self, main_indicator, delimiter='\t', **kwargs):
|
||||
self.delimiter = delimiter
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.results = [] # clear results
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
# NOTE: only support bs=1 now, as the label length of different sample is Unequal
|
||||
assert len(
|
||||
preds) == 1, "CentripetalText test now only suuport batch_size=1."
|
||||
label = batch[2]
|
||||
text = batch[3]
|
||||
pred = preds[0]['points']
|
||||
result = get_score_C(label, text, pred)
|
||||
|
||||
self.results.append(result)
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
Input format: y0,x0, ..... yn,xn. Each detection is separated by the end of line token ('\n')'
|
||||
"""
|
||||
metrics = combine_results(self.results, rec_flag=False)
|
||||
self.reset()
|
||||
return metrics
|
||||
@@ -0,0 +1,154 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
__all__ = ['DetMetric', 'DetFCEMetric']
|
||||
|
||||
from .eval_det_iou import DetectionIoUEvaluator
|
||||
|
||||
|
||||
class DetMetric(object):
|
||||
def __init__(self, main_indicator='hmean', **kwargs):
|
||||
self.evaluator = DetectionIoUEvaluator()
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
'''
|
||||
batch: a list produced by dataloaders.
|
||||
image: np.ndarray of shape (N, C, H, W).
|
||||
ratio_list: np.ndarray of shape(N,2)
|
||||
polygons: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
|
||||
ignore_tags: np.ndarray of shape (N, K), indicates whether a region is ignorable or not.
|
||||
preds: a list of dict produced by post process
|
||||
points: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
|
||||
'''
|
||||
gt_polyons_batch = batch[2]
|
||||
ignore_tags_batch = batch[3]
|
||||
for pred, gt_polyons, ignore_tags in zip(preds, gt_polyons_batch,
|
||||
ignore_tags_batch):
|
||||
# prepare gt
|
||||
gt_info_list = [{
|
||||
'points': gt_polyon,
|
||||
'text': '',
|
||||
'ignore': ignore_tag
|
||||
} for gt_polyon, ignore_tag in zip(gt_polyons, ignore_tags)]
|
||||
# prepare det
|
||||
det_info_list = [{
|
||||
'points': det_polyon,
|
||||
'text': ''
|
||||
} for det_polyon in pred['points']]
|
||||
result = self.evaluator.evaluate_image(gt_info_list, det_info_list)
|
||||
self.results.append(result)
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'precision': 0,
|
||||
'recall': 0,
|
||||
'hmean': 0
|
||||
}
|
||||
"""
|
||||
|
||||
metrics = self.evaluator.combine_results(self.results)
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.results = [] # clear results
|
||||
|
||||
|
||||
class DetFCEMetric(object):
|
||||
def __init__(self, main_indicator='hmean', **kwargs):
|
||||
self.evaluator = DetectionIoUEvaluator()
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
'''
|
||||
batch: a list produced by dataloaders.
|
||||
image: np.ndarray of shape (N, C, H, W).
|
||||
ratio_list: np.ndarray of shape(N,2)
|
||||
polygons: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
|
||||
ignore_tags: np.ndarray of shape (N, K), indicates whether a region is ignorable or not.
|
||||
preds: a list of dict produced by post process
|
||||
points: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
|
||||
'''
|
||||
gt_polyons_batch = batch[2]
|
||||
ignore_tags_batch = batch[3]
|
||||
|
||||
for pred, gt_polyons, ignore_tags in zip(preds, gt_polyons_batch,
|
||||
ignore_tags_batch):
|
||||
# prepare gt
|
||||
gt_info_list = [{
|
||||
'points': gt_polyon,
|
||||
'text': '',
|
||||
'ignore': ignore_tag
|
||||
} for gt_polyon, ignore_tag in zip(gt_polyons, ignore_tags)]
|
||||
# prepare det
|
||||
det_info_list = [{
|
||||
'points': det_polyon,
|
||||
'text': '',
|
||||
'score': score
|
||||
} for det_polyon, score in zip(pred['points'], pred['scores'])]
|
||||
|
||||
for score_thr in self.results.keys():
|
||||
det_info_list_thr = [
|
||||
det_info for det_info in det_info_list
|
||||
if det_info['score'] >= score_thr
|
||||
]
|
||||
result = self.evaluator.evaluate_image(gt_info_list,
|
||||
det_info_list_thr)
|
||||
self.results[score_thr].append(result)
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {'heman':0,
|
||||
'thr 0.3':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.4':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.5':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.6':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.7':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.8':'precision: 0 recall: 0 hmean: 0',
|
||||
'thr 0.9':'precision: 0 recall: 0 hmean: 0',
|
||||
}
|
||||
"""
|
||||
metrics = {}
|
||||
hmean = 0
|
||||
for score_thr in self.results.keys():
|
||||
metric = self.evaluator.combine_results(self.results[score_thr])
|
||||
# for key, value in metric.items():
|
||||
# metrics['{}_{}'.format(key, score_thr)] = value
|
||||
metric_str = 'precision:{:.5f} recall:{:.5f} hmean:{:.5f}'.format(
|
||||
metric['precision'], metric['recall'], metric['hmean'])
|
||||
metrics['thr {}'.format(score_thr)] = metric_str
|
||||
hmean = max(hmean, metric['hmean'])
|
||||
metrics['hmean'] = hmean
|
||||
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.results = {
|
||||
0.3: [],
|
||||
0.4: [],
|
||||
0.5: [],
|
||||
0.6: [],
|
||||
0.7: [],
|
||||
0.8: [],
|
||||
0.9: []
|
||||
} # clear results
|
||||
@@ -0,0 +1,75 @@
|
||||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import importlib
|
||||
import copy
|
||||
|
||||
from .rec_metric import RecMetric
|
||||
from .det_metric import DetMetric
|
||||
from .e2e_metric import E2EMetric
|
||||
from .cls_metric import ClsMetric
|
||||
from .vqa_token_ser_metric import VQASerTokenMetric
|
||||
from .vqa_token_re_metric import VQAReTokenMetric
|
||||
|
||||
|
||||
class DistillationMetric(object):
|
||||
def __init__(self,
|
||||
key=None,
|
||||
base_metric_name=None,
|
||||
main_indicator=None,
|
||||
**kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.key = key
|
||||
self.main_indicator = main_indicator
|
||||
self.base_metric_name = base_metric_name
|
||||
self.kwargs = kwargs
|
||||
self.metrics = None
|
||||
|
||||
def _init_metrcis(self, preds):
|
||||
self.metrics = dict()
|
||||
mod = importlib.import_module(__name__)
|
||||
for key in preds:
|
||||
self.metrics[key] = getattr(mod, self.base_metric_name)(
|
||||
main_indicator=self.main_indicator, **self.kwargs)
|
||||
self.metrics[key].reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
assert isinstance(preds, dict)
|
||||
if self.metrics is None:
|
||||
self._init_metrcis(preds)
|
||||
output = dict()
|
||||
for key in preds:
|
||||
self.metrics[key].__call__(preds[key], batch, **kwargs)
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0,
|
||||
'norm_edit_dis': 0,
|
||||
}
|
||||
"""
|
||||
output = dict()
|
||||
for key in self.metrics:
|
||||
metric = self.metrics[key].get_metric()
|
||||
# main indicator
|
||||
if key == self.key:
|
||||
output.update(metric)
|
||||
else:
|
||||
for sub_key in metric:
|
||||
output["{}_{}".format(key, sub_key)] = metric[sub_key]
|
||||
return output
|
||||
|
||||
def reset(self):
|
||||
for key in self.metrics:
|
||||
self.metrics[key].reset()
|
||||
@@ -0,0 +1,86 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
__all__ = ['E2EMetric']
|
||||
|
||||
from ppocr.utils.e2e_metric.Deteval import get_socre_A, get_socre_B, combine_results
|
||||
from ppocr.utils.e2e_utils.extract_textpoint_slow import get_dict
|
||||
|
||||
|
||||
class E2EMetric(object):
|
||||
def __init__(self,
|
||||
mode,
|
||||
gt_mat_dir,
|
||||
character_dict_path,
|
||||
main_indicator='f_score_e2e',
|
||||
**kwargs):
|
||||
self.mode = mode
|
||||
self.gt_mat_dir = gt_mat_dir
|
||||
self.label_list = get_dict(character_dict_path)
|
||||
self.max_index = len(self.label_list)
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
if self.mode == 'A':
|
||||
gt_polyons_batch = batch[2]
|
||||
temp_gt_strs_batch = batch[3][0]
|
||||
ignore_tags_batch = batch[4]
|
||||
gt_strs_batch = []
|
||||
|
||||
for temp_list in temp_gt_strs_batch:
|
||||
t = ""
|
||||
for index in temp_list:
|
||||
if index < self.max_index:
|
||||
t += self.label_list[index]
|
||||
gt_strs_batch.append(t)
|
||||
|
||||
for pred, gt_polyons, gt_strs, ignore_tags in zip(
|
||||
[preds], gt_polyons_batch, [gt_strs_batch], ignore_tags_batch):
|
||||
# prepare gt
|
||||
gt_info_list = [{
|
||||
'points': gt_polyon,
|
||||
'text': gt_str,
|
||||
'ignore': ignore_tag
|
||||
} for gt_polyon, gt_str, ignore_tag in
|
||||
zip(gt_polyons, gt_strs, ignore_tags)]
|
||||
# prepare det
|
||||
e2e_info_list = [{
|
||||
'points': det_polyon,
|
||||
'texts': pred_str
|
||||
} for det_polyon, pred_str in
|
||||
zip(pred['points'], pred['texts'])]
|
||||
|
||||
result = get_socre_A(gt_info_list, e2e_info_list)
|
||||
self.results.append(result)
|
||||
else:
|
||||
img_id = batch[5][0]
|
||||
e2e_info_list = [{
|
||||
'points': det_polyon,
|
||||
'texts': pred_str
|
||||
} for det_polyon, pred_str in zip(preds['points'], preds['texts'])]
|
||||
result = get_socre_B(self.gt_mat_dir, img_id, e2e_info_list)
|
||||
self.results.append(result)
|
||||
|
||||
def get_metric(self):
|
||||
metrics = combine_results(self.results)
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.results = [] # clear results
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
from shapely.geometry import Polygon
|
||||
"""
|
||||
reference from :
|
||||
https://github.com/MhLiao/DB/blob/3c32b808d4412680310d3d28eeb6a2d5bf1566c5/concern/icdar2015_eval/detection/iou.py#L8
|
||||
"""
|
||||
|
||||
|
||||
class DetectionIoUEvaluator(object):
|
||||
def __init__(self, iou_constraint=0.5, area_precision_constraint=0.5):
|
||||
self.iou_constraint = iou_constraint
|
||||
self.area_precision_constraint = area_precision_constraint
|
||||
|
||||
def evaluate_image(self, gt, pred):
|
||||
def get_union(pD, pG):
|
||||
return Polygon(pD).union(Polygon(pG)).area
|
||||
|
||||
def get_intersection_over_union(pD, pG):
|
||||
return get_intersection(pD, pG) / get_union(pD, pG)
|
||||
|
||||
def get_intersection(pD, pG):
|
||||
return Polygon(pD).intersection(Polygon(pG)).area
|
||||
|
||||
def compute_ap(confList, matchList, numGtCare):
|
||||
correct = 0
|
||||
AP = 0
|
||||
if len(confList) > 0:
|
||||
confList = np.array(confList)
|
||||
matchList = np.array(matchList)
|
||||
sorted_ind = np.argsort(-confList)
|
||||
confList = confList[sorted_ind]
|
||||
matchList = matchList[sorted_ind]
|
||||
for n in range(len(confList)):
|
||||
match = matchList[n]
|
||||
if match:
|
||||
correct += 1
|
||||
AP += float(correct) / (n + 1)
|
||||
|
||||
if numGtCare > 0:
|
||||
AP /= numGtCare
|
||||
|
||||
return AP
|
||||
|
||||
perSampleMetrics = {}
|
||||
|
||||
matchedSum = 0
|
||||
|
||||
Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
|
||||
|
||||
numGlobalCareGt = 0
|
||||
numGlobalCareDet = 0
|
||||
|
||||
arrGlobalConfidences = []
|
||||
arrGlobalMatches = []
|
||||
|
||||
recall = 0
|
||||
precision = 0
|
||||
hmean = 0
|
||||
|
||||
detMatched = 0
|
||||
|
||||
iouMat = np.empty([1, 1])
|
||||
|
||||
gtPols = []
|
||||
detPols = []
|
||||
|
||||
gtPolPoints = []
|
||||
detPolPoints = []
|
||||
|
||||
# Array of Ground Truth Polygons' keys marked as don't Care
|
||||
gtDontCarePolsNum = []
|
||||
# Array of Detected Polygons' matched with a don't Care GT
|
||||
detDontCarePolsNum = []
|
||||
|
||||
pairs = []
|
||||
detMatchedNums = []
|
||||
|
||||
arrSampleConfidences = []
|
||||
arrSampleMatch = []
|
||||
|
||||
evaluationLog = ""
|
||||
|
||||
for n in range(len(gt)):
|
||||
points = gt[n]['points']
|
||||
dontCare = gt[n]['ignore']
|
||||
if not Polygon(points).is_valid:
|
||||
continue
|
||||
|
||||
gtPol = points
|
||||
gtPols.append(gtPol)
|
||||
gtPolPoints.append(points)
|
||||
if dontCare:
|
||||
gtDontCarePolsNum.append(len(gtPols) - 1)
|
||||
|
||||
evaluationLog += "GT polygons: " + str(len(gtPols)) + (
|
||||
" (" + str(len(gtDontCarePolsNum)) + " don't care)\n"
|
||||
if len(gtDontCarePolsNum) > 0 else "\n")
|
||||
|
||||
for n in range(len(pred)):
|
||||
points = pred[n]['points']
|
||||
if not Polygon(points).is_valid:
|
||||
continue
|
||||
|
||||
detPol = points
|
||||
detPols.append(detPol)
|
||||
detPolPoints.append(points)
|
||||
if len(gtDontCarePolsNum) > 0:
|
||||
for dontCarePol in gtDontCarePolsNum:
|
||||
dontCarePol = gtPols[dontCarePol]
|
||||
intersected_area = get_intersection(dontCarePol, detPol)
|
||||
pdDimensions = Polygon(detPol).area
|
||||
precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
|
||||
if (precision > self.area_precision_constraint):
|
||||
detDontCarePolsNum.append(len(detPols) - 1)
|
||||
break
|
||||
|
||||
evaluationLog += "DET polygons: " + str(len(detPols)) + (
|
||||
" (" + str(len(detDontCarePolsNum)) + " don't care)\n"
|
||||
if len(detDontCarePolsNum) > 0 else "\n")
|
||||
|
||||
if len(gtPols) > 0 and len(detPols) > 0:
|
||||
# Calculate IoU and precision matrixs
|
||||
outputShape = [len(gtPols), len(detPols)]
|
||||
iouMat = np.empty(outputShape)
|
||||
gtRectMat = np.zeros(len(gtPols), np.int8)
|
||||
detRectMat = np.zeros(len(detPols), np.int8)
|
||||
for gtNum in range(len(gtPols)):
|
||||
for detNum in range(len(detPols)):
|
||||
pG = gtPols[gtNum]
|
||||
pD = detPols[detNum]
|
||||
iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)
|
||||
|
||||
for gtNum in range(len(gtPols)):
|
||||
for detNum in range(len(detPols)):
|
||||
if gtRectMat[gtNum] == 0 and detRectMat[
|
||||
detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
|
||||
if iouMat[gtNum, detNum] > self.iou_constraint:
|
||||
gtRectMat[gtNum] = 1
|
||||
detRectMat[detNum] = 1
|
||||
detMatched += 1
|
||||
pairs.append({'gt': gtNum, 'det': detNum})
|
||||
detMatchedNums.append(detNum)
|
||||
evaluationLog += "Match GT #" + \
|
||||
str(gtNum) + " with Det #" + str(detNum) + "\n"
|
||||
|
||||
numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
|
||||
numDetCare = (len(detPols) - len(detDontCarePolsNum))
|
||||
if numGtCare == 0:
|
||||
recall = float(1)
|
||||
precision = float(0) if numDetCare > 0 else float(1)
|
||||
else:
|
||||
recall = float(detMatched) / numGtCare
|
||||
precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
|
||||
|
||||
hmean = 0 if (precision + recall) == 0 else 2.0 * \
|
||||
precision * recall / (precision + recall)
|
||||
|
||||
matchedSum += detMatched
|
||||
numGlobalCareGt += numGtCare
|
||||
numGlobalCareDet += numDetCare
|
||||
|
||||
perSampleMetrics = {
|
||||
'gtCare': numGtCare,
|
||||
'detCare': numDetCare,
|
||||
'detMatched': detMatched,
|
||||
}
|
||||
return perSampleMetrics
|
||||
|
||||
def combine_results(self, results):
|
||||
numGlobalCareGt = 0
|
||||
numGlobalCareDet = 0
|
||||
matchedSum = 0
|
||||
for result in results:
|
||||
numGlobalCareGt += result['gtCare']
|
||||
numGlobalCareDet += result['detCare']
|
||||
matchedSum += result['detMatched']
|
||||
|
||||
methodRecall = 0 if numGlobalCareGt == 0 else float(
|
||||
matchedSum) / numGlobalCareGt
|
||||
methodPrecision = 0 if numGlobalCareDet == 0 else float(
|
||||
matchedSum) / numGlobalCareDet
|
||||
methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * \
|
||||
methodRecall * methodPrecision / (
|
||||
methodRecall + methodPrecision)
|
||||
methodMetrics = {
|
||||
'precision': methodPrecision,
|
||||
'recall': methodRecall,
|
||||
'hmean': methodHmean
|
||||
}
|
||||
|
||||
return methodMetrics
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
evaluator = DetectionIoUEvaluator()
|
||||
gts = [[{
|
||||
'points': [(0, 0), (1, 0), (1, 1), (0, 1)],
|
||||
'text': 1234,
|
||||
'ignore': False,
|
||||
}, {
|
||||
'points': [(2, 2), (3, 2), (3, 3), (2, 3)],
|
||||
'text': 5678,
|
||||
'ignore': False,
|
||||
}]]
|
||||
preds = [[{
|
||||
'points': [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
||||
'text': 123,
|
||||
'ignore': False,
|
||||
}]]
|
||||
results = []
|
||||
for gt, pred in zip(gts, preds):
|
||||
results.append(evaluator.evaluate_image(gt, pred))
|
||||
metrics = evaluator.combine_results(results)
|
||||
print(metrics)
|
||||
@@ -0,0 +1,71 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# The code is refer from: https://github.com/open-mmlab/mmocr/blob/main/mmocr/core/evaluation/kie_metric.py
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
|
||||
__all__ = ['KIEMetric']
|
||||
|
||||
|
||||
class KIEMetric(object):
|
||||
def __init__(self, main_indicator='hmean', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
self.node = []
|
||||
self.gt = []
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
nodes, _ = preds
|
||||
gts, tag = batch[4].squeeze(0), batch[5].tolist()[0]
|
||||
gts = gts[:tag[0], :1].reshape([-1])
|
||||
self.node.append(nodes.numpy())
|
||||
self.gt.append(gts)
|
||||
# result = self.compute_f1_score(nodes, gts)
|
||||
# self.results.append(result)
|
||||
|
||||
def compute_f1_score(self, preds, gts):
|
||||
ignores = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 25]
|
||||
C = preds.shape[1]
|
||||
classes = np.array(sorted(set(range(C)) - set(ignores)))
|
||||
hist = np.bincount(
|
||||
(gts * C).astype('int64') + preds.argmax(1), minlength=C
|
||||
**2).reshape([C, C]).astype('float32')
|
||||
diag = np.diag(hist)
|
||||
recalls = diag / hist.sum(1).clip(min=1)
|
||||
precisions = diag / hist.sum(0).clip(min=1)
|
||||
f1 = 2 * recalls * precisions / (recalls + precisions).clip(min=1e-8)
|
||||
return f1[classes]
|
||||
|
||||
def combine_results(self, results):
|
||||
node = np.concatenate(self.node, 0)
|
||||
gts = np.concatenate(self.gt, 0)
|
||||
results = self.compute_f1_score(node, gts)
|
||||
data = {'hmean': results.mean()}
|
||||
return data
|
||||
|
||||
def get_metric(self):
|
||||
|
||||
metrics = self.combine_results(self.results)
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.results = [] # clear results
|
||||
self.node = []
|
||||
self.gt = []
|
||||
@@ -0,0 +1,179 @@
|
||||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from rapidfuzz.distance import Levenshtein
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
import numpy as np
|
||||
import string
|
||||
|
||||
|
||||
class RecMetric(object):
|
||||
def __init__(self,
|
||||
main_indicator='acc',
|
||||
is_filter=False,
|
||||
ignore_space=True,
|
||||
**kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.is_filter = is_filter
|
||||
self.ignore_space = ignore_space
|
||||
self.eps = 1e-5
|
||||
self.reset()
|
||||
|
||||
def _normalize_text(self, text):
|
||||
text = ''.join(
|
||||
filter(lambda x: x in (string.digits + string.ascii_letters), text))
|
||||
return text.lower()
|
||||
|
||||
def __call__(self, pred_label, *args, **kwargs):
|
||||
preds, labels = pred_label
|
||||
correct_num = 0
|
||||
all_num = 0
|
||||
norm_edit_dis = 0.0
|
||||
for (pred, pred_conf), (target, _) in zip(preds, labels):
|
||||
if self.ignore_space:
|
||||
pred = pred.replace(" ", "")
|
||||
target = target.replace(" ", "")
|
||||
if self.is_filter:
|
||||
pred = self._normalize_text(pred)
|
||||
target = self._normalize_text(target)
|
||||
norm_edit_dis += Levenshtein.normalized_distance(pred, target)
|
||||
if pred == target:
|
||||
correct_num += 1
|
||||
all_num += 1
|
||||
self.correct_num += correct_num
|
||||
self.all_num += all_num
|
||||
self.norm_edit_dis += norm_edit_dis
|
||||
return {
|
||||
'acc': correct_num / (all_num + self.eps),
|
||||
'norm_edit_dis': 1 - norm_edit_dis / (all_num + self.eps)
|
||||
}
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0,
|
||||
'norm_edit_dis': 0,
|
||||
}
|
||||
"""
|
||||
acc = 1.0 * self.correct_num / (self.all_num + self.eps)
|
||||
norm_edit_dis = 1 - self.norm_edit_dis / (self.all_num + self.eps)
|
||||
self.reset()
|
||||
return {'acc': acc, 'norm_edit_dis': norm_edit_dis}
|
||||
|
||||
def reset(self):
|
||||
self.correct_num = 0
|
||||
self.all_num = 0
|
||||
self.norm_edit_dis = 0
|
||||
|
||||
|
||||
class CNTMetric(object):
|
||||
def __init__(self, main_indicator='acc', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.eps = 1e-5
|
||||
self.reset()
|
||||
|
||||
def __call__(self, pred_label, *args, **kwargs):
|
||||
preds, labels = pred_label
|
||||
correct_num = 0
|
||||
all_num = 0
|
||||
for pred, target in zip(preds, labels):
|
||||
if pred == target:
|
||||
correct_num += 1
|
||||
all_num += 1
|
||||
self.correct_num += correct_num
|
||||
self.all_num += all_num
|
||||
return {'acc': correct_num / (all_num + self.eps), }
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0,
|
||||
}
|
||||
"""
|
||||
acc = 1.0 * self.correct_num / (self.all_num + self.eps)
|
||||
self.reset()
|
||||
return {'acc': acc}
|
||||
|
||||
def reset(self):
|
||||
self.correct_num = 0
|
||||
self.all_num = 0
|
||||
|
||||
|
||||
class CANMetric(object):
|
||||
def __init__(self, main_indicator='exp_rate', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.word_right = []
|
||||
self.exp_right = []
|
||||
self.word_total_length = 0
|
||||
self.exp_total_num = 0
|
||||
self.word_rate = 0
|
||||
self.exp_rate = 0
|
||||
self.reset()
|
||||
self.epoch_reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
epoch_reset = v
|
||||
if epoch_reset:
|
||||
self.epoch_reset()
|
||||
word_probs = preds
|
||||
word_label, word_label_mask = batch
|
||||
line_right = 0
|
||||
if word_probs is not None:
|
||||
word_pred = word_probs.argmax(2)
|
||||
word_pred = word_pred.cpu().detach().numpy()
|
||||
word_scores = [
|
||||
SequenceMatcher(
|
||||
None,
|
||||
s1[:int(np.sum(s3))],
|
||||
s2[:int(np.sum(s3))],
|
||||
autojunk=False).ratio() * (
|
||||
len(s1[:int(np.sum(s3))]) + len(s2[:int(np.sum(s3))])) /
|
||||
len(s1[:int(np.sum(s3))]) / 2
|
||||
for s1, s2, s3 in zip(word_label, word_pred, word_label_mask)
|
||||
]
|
||||
batch_size = len(word_scores)
|
||||
for i in range(batch_size):
|
||||
if word_scores[i] == 1:
|
||||
line_right += 1
|
||||
self.word_rate = np.mean(word_scores) #float
|
||||
self.exp_rate = line_right / batch_size #float
|
||||
exp_length, word_length = word_label.shape[:2]
|
||||
self.word_right.append(self.word_rate * word_length)
|
||||
self.exp_right.append(self.exp_rate * exp_length)
|
||||
self.word_total_length = self.word_total_length + word_length
|
||||
self.exp_total_num = self.exp_total_num + exp_length
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return {
|
||||
'word_rate': 0,
|
||||
"exp_rate": 0,
|
||||
}
|
||||
"""
|
||||
cur_word_rate = sum(self.word_right) / self.word_total_length
|
||||
cur_exp_rate = sum(self.exp_right) / self.exp_total_num
|
||||
self.reset()
|
||||
return {'word_rate': cur_word_rate, "exp_rate": cur_exp_rate}
|
||||
|
||||
def reset(self):
|
||||
self.word_rate = 0
|
||||
self.exp_rate = 0
|
||||
|
||||
def epoch_reset(self):
|
||||
self.word_right = []
|
||||
self.exp_right = []
|
||||
self.word_total_length = 0
|
||||
self.exp_total_num = 0
|
||||
@@ -0,0 +1,155 @@
|
||||
# copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
https://github.com/FudanVI/FudanOCR/blob/main/text-gestalt/utils/ssim_psnr.py
|
||||
"""
|
||||
|
||||
from math import exp
|
||||
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddle.nn as nn
|
||||
import string
|
||||
|
||||
|
||||
class SSIM(nn.Layer):
|
||||
def __init__(self, window_size=11, size_average=True):
|
||||
super(SSIM, self).__init__()
|
||||
self.window_size = window_size
|
||||
self.size_average = size_average
|
||||
self.channel = 1
|
||||
self.window = self.create_window(window_size, self.channel)
|
||||
|
||||
def gaussian(self, window_size, sigma):
|
||||
gauss = paddle.to_tensor([
|
||||
exp(-(x - window_size // 2)**2 / float(2 * sigma**2))
|
||||
for x in range(window_size)
|
||||
])
|
||||
return gauss / gauss.sum()
|
||||
|
||||
def create_window(self, window_size, channel):
|
||||
_1D_window = self.gaussian(window_size, 1.5).unsqueeze(1)
|
||||
_2D_window = _1D_window.mm(_1D_window.t()).unsqueeze(0).unsqueeze(0)
|
||||
window = _2D_window.expand([channel, 1, window_size, window_size])
|
||||
return window
|
||||
|
||||
def _ssim(self, img1, img2, window, window_size, channel,
|
||||
size_average=True):
|
||||
mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel)
|
||||
mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel)
|
||||
|
||||
mu1_sq = mu1.pow(2)
|
||||
mu2_sq = mu2.pow(2)
|
||||
mu1_mu2 = mu1 * mu2
|
||||
|
||||
sigma1_sq = F.conv2d(
|
||||
img1 * img1, window, padding=window_size // 2,
|
||||
groups=channel) - mu1_sq
|
||||
sigma2_sq = F.conv2d(
|
||||
img2 * img2, window, padding=window_size // 2,
|
||||
groups=channel) - mu2_sq
|
||||
sigma12 = F.conv2d(
|
||||
img1 * img2, window, padding=window_size // 2,
|
||||
groups=channel) - mu1_mu2
|
||||
|
||||
C1 = 0.01**2
|
||||
C2 = 0.03**2
|
||||
|
||||
ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / (
|
||||
(mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2))
|
||||
|
||||
if size_average:
|
||||
return ssim_map.mean()
|
||||
else:
|
||||
return ssim_map.mean([1, 2, 3])
|
||||
|
||||
def ssim(self, img1, img2, window_size=11, size_average=True):
|
||||
(_, channel, _, _) = img1.shape
|
||||
window = self.create_window(window_size, channel)
|
||||
|
||||
return self._ssim(img1, img2, window, window_size, channel,
|
||||
size_average)
|
||||
|
||||
def forward(self, img1, img2):
|
||||
(_, channel, _, _) = img1.shape
|
||||
|
||||
if channel == self.channel and self.window.dtype == img1.dtype:
|
||||
window = self.window
|
||||
else:
|
||||
window = self.create_window(self.window_size, channel)
|
||||
|
||||
self.window = window
|
||||
self.channel = channel
|
||||
|
||||
return self._ssim(img1, img2, window, self.window_size, channel,
|
||||
self.size_average)
|
||||
|
||||
|
||||
class SRMetric(object):
|
||||
def __init__(self, main_indicator='all', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.eps = 1e-5
|
||||
self.psnr_result = []
|
||||
self.ssim_result = []
|
||||
self.calculate_ssim = SSIM()
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.correct_num = 0
|
||||
self.all_num = 0
|
||||
self.norm_edit_dis = 0
|
||||
self.psnr_result = []
|
||||
self.ssim_result = []
|
||||
|
||||
def calculate_psnr(self, img1, img2):
|
||||
# img1 and img2 have range [0, 1]
|
||||
mse = ((img1 * 255 - img2 * 255)**2).mean()
|
||||
if mse == 0:
|
||||
return float('inf')
|
||||
return 20 * paddle.log10(255.0 / paddle.sqrt(mse))
|
||||
|
||||
def _normalize_text(self, text):
|
||||
text = ''.join(
|
||||
filter(lambda x: x in (string.digits + string.ascii_letters), text))
|
||||
return text.lower()
|
||||
|
||||
def __call__(self, pred_label, *args, **kwargs):
|
||||
metric = {}
|
||||
images_sr = pred_label["sr_img"]
|
||||
images_hr = pred_label["hr_img"]
|
||||
psnr = self.calculate_psnr(images_sr, images_hr)
|
||||
ssim = self.calculate_ssim(images_sr, images_hr)
|
||||
self.psnr_result.append(psnr)
|
||||
self.ssim_result.append(ssim)
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0,
|
||||
'norm_edit_dis': 0,
|
||||
}
|
||||
"""
|
||||
self.psnr_avg = sum(self.psnr_result) / len(self.psnr_result)
|
||||
self.psnr_avg = round(self.psnr_avg.item(), 6)
|
||||
self.ssim_avg = sum(self.ssim_result) / len(self.ssim_result)
|
||||
self.ssim_avg = round(self.ssim_avg.item(), 6)
|
||||
|
||||
self.all_avg = self.psnr_avg + self.ssim_avg
|
||||
|
||||
self.reset()
|
||||
return {
|
||||
'psnr_avg': self.psnr_avg,
|
||||
"ssim_avg": self.ssim_avg,
|
||||
"all": self.all_avg
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import numpy as np
|
||||
from ppocr.metrics.det_metric import DetMetric
|
||||
|
||||
|
||||
class TableStructureMetric(object):
|
||||
def __init__(self,
|
||||
main_indicator='acc',
|
||||
eps=1e-6,
|
||||
del_thead_tbody=False,
|
||||
**kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.eps = eps
|
||||
self.del_thead_tbody = del_thead_tbody
|
||||
self.reset()
|
||||
|
||||
def __call__(self, pred_label, batch=None, *args, **kwargs):
|
||||
preds, labels = pred_label
|
||||
pred_structure_batch_list = preds['structure_batch_list']
|
||||
gt_structure_batch_list = labels['structure_batch_list']
|
||||
correct_num = 0
|
||||
all_num = 0
|
||||
for (pred, pred_conf), target in zip(pred_structure_batch_list,
|
||||
gt_structure_batch_list):
|
||||
pred_str = ''.join(pred)
|
||||
target_str = ''.join(target)
|
||||
if self.del_thead_tbody:
|
||||
pred_str = pred_str.replace('<thead>', '').replace(
|
||||
'</thead>', '').replace('<tbody>', '').replace('</tbody>',
|
||||
'')
|
||||
target_str = target_str.replace('<thead>', '').replace(
|
||||
'</thead>', '').replace('<tbody>', '').replace('</tbody>',
|
||||
'')
|
||||
if pred_str == target_str:
|
||||
correct_num += 1
|
||||
all_num += 1
|
||||
self.correct_num += correct_num
|
||||
self.all_num += all_num
|
||||
|
||||
def get_metric(self):
|
||||
"""
|
||||
return metrics {
|
||||
'acc': 0,
|
||||
}
|
||||
"""
|
||||
acc = 1.0 * self.correct_num / (self.all_num + self.eps)
|
||||
self.reset()
|
||||
return {'acc': acc}
|
||||
|
||||
def reset(self):
|
||||
self.correct_num = 0
|
||||
self.all_num = 0
|
||||
self.len_acc_num = 0
|
||||
self.token_nums = 0
|
||||
self.anys_dict = dict()
|
||||
|
||||
|
||||
class TableMetric(object):
|
||||
def __init__(self,
|
||||
main_indicator='acc',
|
||||
compute_bbox_metric=False,
|
||||
box_format='xyxy',
|
||||
del_thead_tbody=False,
|
||||
**kwargs):
|
||||
"""
|
||||
|
||||
@param sub_metrics: configs of sub_metric
|
||||
@param main_matric: main_matric for save best_model
|
||||
@param kwargs:
|
||||
"""
|
||||
self.structure_metric = TableStructureMetric(
|
||||
del_thead_tbody=del_thead_tbody)
|
||||
self.bbox_metric = DetMetric() if compute_bbox_metric else None
|
||||
self.main_indicator = main_indicator
|
||||
self.box_format = box_format
|
||||
self.reset()
|
||||
|
||||
def __call__(self, pred_label, batch=None, *args, **kwargs):
|
||||
self.structure_metric(pred_label)
|
||||
if self.bbox_metric is not None:
|
||||
self.bbox_metric(*self.prepare_bbox_metric_input(pred_label))
|
||||
|
||||
def prepare_bbox_metric_input(self, pred_label):
|
||||
pred_bbox_batch_list = []
|
||||
gt_ignore_tags_batch_list = []
|
||||
gt_bbox_batch_list = []
|
||||
preds, labels = pred_label
|
||||
|
||||
batch_num = len(preds['bbox_batch_list'])
|
||||
for batch_idx in range(batch_num):
|
||||
# pred
|
||||
pred_bbox_list = [
|
||||
self.format_box(pred_box)
|
||||
for pred_box in preds['bbox_batch_list'][batch_idx]
|
||||
]
|
||||
pred_bbox_batch_list.append({'points': pred_bbox_list})
|
||||
|
||||
# gt
|
||||
gt_bbox_list = []
|
||||
gt_ignore_tags_list = []
|
||||
for gt_box in labels['bbox_batch_list'][batch_idx]:
|
||||
gt_bbox_list.append(self.format_box(gt_box))
|
||||
gt_ignore_tags_list.append(0)
|
||||
gt_bbox_batch_list.append(gt_bbox_list)
|
||||
gt_ignore_tags_batch_list.append(gt_ignore_tags_list)
|
||||
|
||||
return [
|
||||
pred_bbox_batch_list,
|
||||
[0, 0, gt_bbox_batch_list, gt_ignore_tags_batch_list]
|
||||
]
|
||||
|
||||
def get_metric(self):
|
||||
structure_metric = self.structure_metric.get_metric()
|
||||
if self.bbox_metric is None:
|
||||
return structure_metric
|
||||
bbox_metric = self.bbox_metric.get_metric()
|
||||
if self.main_indicator == self.bbox_metric.main_indicator:
|
||||
output = bbox_metric
|
||||
for sub_key in structure_metric:
|
||||
output["structure_metric_{}".format(
|
||||
sub_key)] = structure_metric[sub_key]
|
||||
else:
|
||||
output = structure_metric
|
||||
for sub_key in bbox_metric:
|
||||
output["bbox_metric_{}".format(sub_key)] = bbox_metric[sub_key]
|
||||
return output
|
||||
|
||||
def reset(self):
|
||||
self.structure_metric.reset()
|
||||
if self.bbox_metric is not None:
|
||||
self.bbox_metric.reset()
|
||||
|
||||
def format_box(self, box):
|
||||
if self.box_format == 'xyxy':
|
||||
x1, y1, x2, y2 = box
|
||||
box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
|
||||
elif self.box_format == 'xywh':
|
||||
x, y, w, h = box
|
||||
x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
|
||||
box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
|
||||
elif self.box_format == 'xyxyxyxy':
|
||||
x1, y1, x2, y2, x3, y3, x4, y4 = box
|
||||
box = [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
|
||||
return box
|
||||
@@ -0,0 +1,181 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
|
||||
__all__ = ['KIEMetric']
|
||||
|
||||
|
||||
class VQAReTokenMetric(object):
|
||||
def __init__(self, main_indicator='hmean', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
pred_relations, relations, entities = preds
|
||||
self.pred_relations_list.extend(pred_relations)
|
||||
self.relations_list.extend(relations)
|
||||
self.entities_list.extend(entities)
|
||||
|
||||
def get_metric(self):
|
||||
gt_relations = []
|
||||
for b in range(len(self.relations_list)):
|
||||
rel_sent = []
|
||||
relation_list = self.relations_list[b]
|
||||
entitie_list = self.entities_list[b]
|
||||
head_len = relation_list[0, 0]
|
||||
if head_len > 0:
|
||||
entitie_start_list = entitie_list[1:entitie_list[0, 0] + 1, 0]
|
||||
entitie_end_list = entitie_list[1:entitie_list[0, 1] + 1, 1]
|
||||
entitie_label_list = entitie_list[1:entitie_list[0, 2] + 1, 2]
|
||||
for head, tail in zip(relation_list[1:head_len + 1, 0],
|
||||
relation_list[1:head_len + 1, 1]):
|
||||
rel = {}
|
||||
rel["head_id"] = head
|
||||
rel["head"] = (entitie_start_list[head],
|
||||
entitie_end_list[head])
|
||||
rel["head_type"] = entitie_label_list[head]
|
||||
|
||||
rel["tail_id"] = tail
|
||||
rel["tail"] = (entitie_start_list[tail],
|
||||
entitie_end_list[tail])
|
||||
rel["tail_type"] = entitie_label_list[tail]
|
||||
|
||||
rel["type"] = 1
|
||||
rel_sent.append(rel)
|
||||
gt_relations.append(rel_sent)
|
||||
re_metrics = self.re_score(
|
||||
self.pred_relations_list, gt_relations, mode="boundaries")
|
||||
metrics = {
|
||||
"precision": re_metrics["ALL"]["p"],
|
||||
"recall": re_metrics["ALL"]["r"],
|
||||
"hmean": re_metrics["ALL"]["f1"],
|
||||
}
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.pred_relations_list = []
|
||||
self.relations_list = []
|
||||
self.entities_list = []
|
||||
|
||||
def re_score(self, pred_relations, gt_relations, mode="strict"):
|
||||
"""Evaluate RE predictions
|
||||
|
||||
Args:
|
||||
pred_relations (list) : list of list of predicted relations (several relations in each sentence)
|
||||
gt_relations (list) : list of list of ground truth relations
|
||||
|
||||
rel = { "head": (start_idx (inclusive), end_idx (exclusive)),
|
||||
"tail": (start_idx (inclusive), end_idx (exclusive)),
|
||||
"head_type": ent_type,
|
||||
"tail_type": ent_type,
|
||||
"type": rel_type}
|
||||
|
||||
vocab (Vocab) : dataset vocabulary
|
||||
mode (str) : in 'strict' or 'boundaries'"""
|
||||
|
||||
assert mode in ["strict", "boundaries"]
|
||||
|
||||
relation_types = [v for v in [0, 1] if not v == 0]
|
||||
scores = {
|
||||
rel: {
|
||||
"tp": 0,
|
||||
"fp": 0,
|
||||
"fn": 0
|
||||
}
|
||||
for rel in relation_types + ["ALL"]
|
||||
}
|
||||
|
||||
# Count GT relations and Predicted relations
|
||||
n_sents = len(gt_relations)
|
||||
n_rels = sum([len([rel for rel in sent]) for sent in gt_relations])
|
||||
n_found = sum([len([rel for rel in sent]) for sent in pred_relations])
|
||||
|
||||
# Count TP, FP and FN per type
|
||||
for pred_sent, gt_sent in zip(pred_relations, gt_relations):
|
||||
for rel_type in relation_types:
|
||||
# strict mode takes argument types into account
|
||||
if mode == "strict":
|
||||
pred_rels = {(rel["head"], rel["head_type"], rel["tail"],
|
||||
rel["tail_type"])
|
||||
for rel in pred_sent
|
||||
if rel["type"] == rel_type}
|
||||
gt_rels = {(rel["head"], rel["head_type"], rel["tail"],
|
||||
rel["tail_type"])
|
||||
for rel in gt_sent if rel["type"] == rel_type}
|
||||
|
||||
# boundaries mode only takes argument spans into account
|
||||
elif mode == "boundaries":
|
||||
pred_rels = {(rel["head"], rel["tail"])
|
||||
for rel in pred_sent
|
||||
if rel["type"] == rel_type}
|
||||
gt_rels = {(rel["head"], rel["tail"])
|
||||
for rel in gt_sent if rel["type"] == rel_type}
|
||||
|
||||
scores[rel_type]["tp"] += len(pred_rels & gt_rels)
|
||||
scores[rel_type]["fp"] += len(pred_rels - gt_rels)
|
||||
scores[rel_type]["fn"] += len(gt_rels - pred_rels)
|
||||
|
||||
# Compute per entity Precision / Recall / F1
|
||||
for rel_type in scores.keys():
|
||||
if scores[rel_type]["tp"]:
|
||||
scores[rel_type]["p"] = scores[rel_type]["tp"] / (
|
||||
scores[rel_type]["fp"] + scores[rel_type]["tp"])
|
||||
scores[rel_type]["r"] = scores[rel_type]["tp"] / (
|
||||
scores[rel_type]["fn"] + scores[rel_type]["tp"])
|
||||
else:
|
||||
scores[rel_type]["p"], scores[rel_type]["r"] = 0, 0
|
||||
|
||||
if not scores[rel_type]["p"] + scores[rel_type]["r"] == 0:
|
||||
scores[rel_type]["f1"] = (
|
||||
2 * scores[rel_type]["p"] * scores[rel_type]["r"] /
|
||||
(scores[rel_type]["p"] + scores[rel_type]["r"]))
|
||||
else:
|
||||
scores[rel_type]["f1"] = 0
|
||||
|
||||
# Compute micro F1 Scores
|
||||
tp = sum([scores[rel_type]["tp"] for rel_type in relation_types])
|
||||
fp = sum([scores[rel_type]["fp"] for rel_type in relation_types])
|
||||
fn = sum([scores[rel_type]["fn"] for rel_type in relation_types])
|
||||
|
||||
if tp:
|
||||
precision = tp / (tp + fp)
|
||||
recall = tp / (tp + fn)
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
|
||||
else:
|
||||
precision, recall, f1 = 0, 0, 0
|
||||
|
||||
scores["ALL"]["p"] = precision
|
||||
scores["ALL"]["r"] = recall
|
||||
scores["ALL"]["f1"] = f1
|
||||
scores["ALL"]["tp"] = tp
|
||||
scores["ALL"]["fp"] = fp
|
||||
scores["ALL"]["fn"] = fn
|
||||
|
||||
# Compute Macro F1 Scores
|
||||
scores["ALL"]["Macro_f1"] = np.mean(
|
||||
[scores[ent_type]["f1"] for ent_type in relation_types])
|
||||
scores["ALL"]["Macro_p"] = np.mean(
|
||||
[scores[ent_type]["p"] for ent_type in relation_types])
|
||||
scores["ALL"]["Macro_r"] = np.mean(
|
||||
[scores[ent_type]["r"] for ent_type in relation_types])
|
||||
|
||||
return scores
|
||||
@@ -0,0 +1,47 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
|
||||
__all__ = ['KIEMetric']
|
||||
|
||||
|
||||
class VQASerTokenMetric(object):
|
||||
def __init__(self, main_indicator='hmean', **kwargs):
|
||||
self.main_indicator = main_indicator
|
||||
self.reset()
|
||||
|
||||
def __call__(self, preds, batch, **kwargs):
|
||||
preds, labels = preds
|
||||
self.pred_list.extend(preds)
|
||||
self.gt_list.extend(labels)
|
||||
|
||||
def get_metric(self):
|
||||
from seqeval.metrics import f1_score, precision_score, recall_score
|
||||
metrics = {
|
||||
"precision": precision_score(self.gt_list, self.pred_list),
|
||||
"recall": recall_score(self.gt_list, self.pred_list),
|
||||
"hmean": f1_score(self.gt_list, self.pred_list),
|
||||
}
|
||||
self.reset()
|
||||
return metrics
|
||||
|
||||
def reset(self):
|
||||
self.pred_list = []
|
||||
self.gt_list = []
|
||||
Reference in New Issue
Block a user