Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Diegodlh
Web2Cit Core
Commits
4072efda
Commit
4072efda
authored
Apr 05, 2022
by
Diegodlh
Browse files
Merge branch 'dev' into domain
parents
45a2e357
1ef70d22
Changes
9
Hide whitespace changes
Inline
Side-by-side
src/domain/domainConfiguration.ts
View file @
4072efda
...
...
@@ -27,8 +27,7 @@ export abstract class DomainConfiguration<
revisions
:
Promise
<
RevisionMetadata
[]
>
|
undefined
;
revisionCache
:
Map
<
RevisionMetadata
[
"
revid
"
],
|
Promise
<
ConfigurationRevision
<
ConfigurationDefinitionType
>
|
undefined
>
|
undefined
Promise
<
ContentRevision
|
undefined
>
|
undefined
>
=
new
Map
();
currentRevid
:
RevisionMetadata
[
"
revid
"
]
|
undefined
;
...
...
@@ -92,7 +91,7 @@ export abstract class DomainConfiguration<
private
async
fetchRevision
(
revid
?:
RevisionMetadata
[
"
revid
"
]
):
Promise
<
Con
figurationRevision
<
ConfigurationDefinitionType
>
|
undefined
>
{
):
Promise
<
Con
tentRevision
|
undefined
>
{
const
api
=
new
RevisionsApi
(
this
.
mediawiki
.
instance
);
const
revisions
=
await
api
.
fetchRevisions
(
this
.
title
,
true
,
revid
,
1
);
const
revision
=
revisions
[
0
];
...
...
@@ -101,31 +100,9 @@ export abstract class DomainConfiguration<
let
info
=
`No revision found for page "
${
this
.
title
}
"`
;
if
(
revid
!==
undefined
)
info
=
info
+
` and revid
${
revid
}
`
;
log
.
info
(
info
);
return
undefined
;
}
if
(
revision
.
content
===
undefined
)
{
return
Promise
.
reject
(
`Unexpected undefined revision content`
);
}
const
strippedContent
=
revision
.
content
.
replace
(
'
<syntaxhighlight lang="json">
'
,
""
)
.
replace
(
"
</syntaxhighlight>
"
,
""
);
let
configuration
:
ConfigurationDefinitionType
[];
try
{
configuration
=
this
.
parse
(
strippedContent
);
}
catch
(
e
)
{
let
message
=
"
Could not parse revision content
"
;
if
(
e
instanceof
Error
)
message
=
message
+
`:
${
e
.
message
}
`
;
throw
new
ContentRevisionError
(
message
,
revision
);
}
return
{
revid
:
revision
.
revid
,
timestamp
:
revision
.
timestamp
,
configuration
:
configuration
,
};
return
revision
;
}
getRevisionIds
(
refresh
=
false
):
Promise
<
RevisionMetadata
[]
>
{
...
...
@@ -138,7 +115,7 @@ export abstract class DomainConfiguration<
getRevision
(
revid
:
RevisionMetadata
[
"
revid
"
],
refresh
=
false
):
Promise
<
Con
figurationRevision
<
ConfigurationDefinitionType
>
|
undefined
>
{
):
Promise
<
Con
tentRevision
|
undefined
>
{
let
revisionPromise
=
this
.
revisionCache
.
get
(
revid
);
if
(
revisionPromise
===
undefined
||
refresh
)
{
revisionPromise
=
this
.
fetchRevision
(
revid
);
...
...
@@ -147,9 +124,7 @@ export abstract class DomainConfiguration<
return
revisionPromise
;
}
getLatestRevision
():
Promise
<
ConfigurationRevision
<
ConfigurationDefinitionType
>
|
undefined
>
{
getLatestRevision
():
Promise
<
ContentRevision
|
undefined
>
{
return
this
.
fetchRevision
().
then
((
revision
)
=>
{
if
(
revision
!==
undefined
)
{
const
revisionPromise
=
Promise
.
resolve
(
revision
);
...
...
@@ -159,10 +134,28 @@ export abstract class DomainConfiguration<
});
}
loadRevision
(
revision
:
ConfigurationRevision
<
ConfigurationDefinitionType
>
):
void
{
this
.
loadConfiguration
(
revision
.
configuration
);
loadRevision
(
revision
:
ContentRevision
):
void
{
if
(
revision
.
content
===
undefined
)
{
throw
new
ContentRevisionError
(
`Unexpected undefined revision content`
,
revision
);
}
const
strippedContent
=
revision
.
content
.
replace
(
'
<syntaxhighlight lang="json">
'
,
""
)
.
replace
(
"
</syntaxhighlight>
"
,
""
);
let
configuration
:
ConfigurationDefinitionType
[];
try
{
configuration
=
this
.
parse
(
strippedContent
);
}
catch
(
e
)
{
let
message
=
"
Could not parse revision content
"
;
if
(
e
instanceof
Error
)
message
=
message
+
`:
${
e
.
message
}
`
;
throw
new
ContentRevisionError
(
message
,
revision
);
}
this
.
loadConfiguration
(
configuration
);
this
.
currentRevid
=
revision
.
revid
;
}
...
...
@@ -185,10 +178,6 @@ ${this.toJSON()}
}
}
interface
ConfigurationRevision
<
T
>
extends
RevisionMetadata
{
configuration
:
T
[];
}
class
ContentRevisionError
extends
Error
{
revision
:
ContentRevision
;
constructor
(
message
:
string
,
revision
:
ContentRevision
)
{
...
...
src/domain/templateConfiguration.test.ts
View file @
4072efda
...
...
@@ -7,6 +7,8 @@ import {
import
{
Webpage
}
from
"
../webpage/webpage
"
;
import
*
as
nodeFetch
from
"
node-fetch
"
;
import
{
pages
}
from
"
../webpage/samplePages
"
;
import
log
from
"
loglevel
"
;
import
{
ContentRevision
}
from
"
../mediawiki/revisions
"
;
const
mockNodeFetch
=
nodeFetch
as
typeof
import
(
"
../../__mocks__/node-fetch
"
);
...
...
@@ -273,3 +275,128 @@ it("multiple templates for the same path are silently ignored", () => {
);
expect
(
configuration
.
get
().
length
).
toBe
(
1
);
});
describe
(
"
Configuration revisions
"
,
()
=>
{
const
warnSpy
=
jest
.
spyOn
(
log
,
"
warn
"
).
mockImplementation
();
beforeEach
(()
=>
{
jest
.
clearAllMocks
();
});
it
(
"
skips misformatted elements individually
"
,
()
=>
{
const
content
=
JSON
.
stringify
([
{
path
:
"
/
"
,
fields
:
[
{
fieldname
:
"
itemType
"
,
required
:
true
,
procedures
:
[
{
selections
:
[
{
type
:
"
citoid
"
,
config
:
"
itemType
"
,
},
{
type
:
"
invalidType
"
,
config
:
"
itemType
"
,
},
],
transformations
:
[
{
type
:
"
range
"
,
config
:
"
0
"
,
itemwise
:
true
,
},
{
type
:
"
range
"
,
config
:
"
invalidConfig
"
,
itemwise
:
true
,
},
],
},
{
selections
:
[],
},
{
transformations
:
[],
},
],
},
{
fieldname
:
"
invalidField
"
,
required
:
true
,
procedures
:
[],
},
],
},
]);
const
revision
:
ContentRevision
=
{
revid
:
0
,
timestamp
:
""
,
content
,
};
const
configuration
=
new
TemplateConfiguration
(
"
example.com
"
,
[],
undefined
);
configuration
.
loadRevision
(
revision
);
expect
(
warnSpy
).
toHaveBeenCalledTimes
(
5
);
expect
(
configuration
.
get
().
map
((
template
)
=>
template
.
toJSON
())).
toEqual
([
{
path
:
"
/
"
,
label
:
""
,
fields
:
[
{
fieldname
:
"
itemType
"
,
required
:
true
,
procedures
:
[
{
selections
:
[
{
type
:
"
citoid
"
,
config
:
"
itemType
"
,
},
],
transformations
:
[
{
type
:
"
range
"
,
config
:
"
0
"
,
itemwise
:
true
,
},
],
},
],
},
],
},
]);
});
it
(
"
skips templates with missing mandatory fields
"
,
()
=>
{
const
definitions
:
TemplateDefinition
[]
=
[
{
path
:
"
/
"
,
fields
:
[
{
fieldname
:
"
itemType
"
,
required
:
true
,
procedures
:
[],
},
],
},
];
const
revision
:
ContentRevision
=
{
revid
:
0
,
timestamp
:
""
,
content
:
JSON
.
stringify
(
definitions
),
};
const
configuration
=
new
TemplateConfiguration
(
"
example.com
"
,
[
"
itemType
"
,
"
title
"
],
undefined
);
configuration
.
loadRevision
(
revision
);
expect
(
warnSpy
).
toHaveBeenCalledTimes
(
1
);
expect
(
configuration
.
get
().
length
).
toBe
(
0
);
});
});
src/domain/templateConfiguration.ts
View file @
4072efda
...
...
@@ -9,7 +9,6 @@ import log from "loglevel";
import
{
Webpage
}
from
"
../webpage/webpage
"
;
import
{
FallbackTemplateDefinition
,
isTemplateDefinition
,
TemplateDefinition
,
TemplateOutput
,
}
from
"
../types
"
;
...
...
@@ -80,11 +79,9 @@ export class TemplateConfiguration extends DomainConfiguration<
add
(
definition
:
TemplateDefinition
,
index
?:
number
):
TranslationTemplate
{
// create template instance before checking if path already exists
// because the template constructor may make changes to the path
const
newTemplate
=
new
TranslationTemplate
(
this
.
domain
,
definition
,
this
.
mandatoryFields
);
const
newTemplate
=
new
TranslationTemplate
(
this
.
domain
,
definition
,
{
forceRequiredFields
:
this
.
mandatoryFields
,
});
if
(
this
.
templates
.
some
((
template
)
=>
template
.
path
===
newTemplate
.
path
))
{
throw
new
DuplicateTemplatePathError
(
definition
.
path
);
}
else
{
...
...
@@ -135,19 +132,23 @@ export class TemplateConfiguration extends DomainConfiguration<
}
const
templateDefinitions
=
definitions
.
reduce
(
(
templateDefinitions
:
TemplateDefinition
[],
definition
,
index
)
=>
{
// fixme: instead of ignoring templates
// with unsupported fields or selection/transformation steps
// simply ignore the unsupported objects
if
(
isTemplateDefinition
(
definition
))
{
templateDefinitions
.
push
(
definition
);
}
else
{
// to address T305267, instead of using isTemplateDefinition,
// create template from definition, skipping individual invalid elements
// and convert back to json
try
{
const
template
=
new
TranslationTemplate
(
this
.
domain
,
definition
,
{
forceRequiredFields
:
this
.
mandatoryFields
,
strict
:
false
,
});
templateDefinitions
.
push
(
template
.
toJSON
());
}
catch
(
error
)
{
let
info
=
"
Ignoring misformatted template
"
;
if
(
"
path
"
in
definition
)
{
info
=
info
+
` for path "
${
definition
.
path
}
"`
;
}
else
{
info
=
info
+
` at index
${
index
}
`
;
}
log
.
info
(
info
);
log
.
warn
(
info
+
`:
${
error
}
`
);
}
return
templateDefinitions
;
},
...
...
@@ -157,15 +158,9 @@ export class TemplateConfiguration extends DomainConfiguration<
}
loadConfiguration
(
templates
:
TemplateDefinition
[]):
void
{
if
(
this
.
mandatoryFields
===
undefined
)
{
throw
new
Error
(
"
Mandatory template fields must be defined before loading any template configuration
"
);
}
// fixme?: wiping previous translation templates erases template caches
// see T302239
this
.
templates
=
[];
// silently ignore duplicate
templates
.
forEach
((
definition
)
=>
{
try
{
this
.
add
(
definition
);
...
...
src/templates/procedure.test.ts
View file @
4072efda
...
...
@@ -4,6 +4,8 @@ import { CitoidSelection } from "./selection";
import
{
JoinTransformation
,
RangeTransformation
}
from
"
./transformation
"
;
import
*
as
nodeFetch
from
"
node-fetch
"
;
import
{
pages
}
from
"
../webpage/samplePages
"
;
import
log
from
"
loglevel
"
;
import
{
ProcedureDefinition
}
from
"
../types
"
;
const
mockNodeFetch
=
nodeFetch
as
typeof
import
(
"
../../__mocks__/node-fetch
"
);
...
...
@@ -66,4 +68,61 @@ it("does not return selection output if transformation output is an empty array"
});
});
it
(
"
constructor optionally skips invalid translation step definitions
"
,
()
=>
{
const
warnSpy
=
jest
.
spyOn
(
log
,
"
warn
"
).
mockImplementation
();
const
definition
:
ProcedureDefinition
=
{
selections
:
[
{
type
:
"
citoid
"
,
config
:
"
itemType
"
,
},
{
type
:
"
citoid
"
,
config
:
"
invalidConfig
"
,
},
{
type
:
"
invalidType
"
,
config
:
"
itemType
"
,
},
],
transformations
:
[
{
type
:
"
range
"
,
config
:
"
0
"
,
itemwise
:
true
,
},
{
type
:
"
invalidType
"
,
config
:
"
0
"
,
itemwise
:
true
,
},
{
type
:
"
range
"
,
config
:
"
invalidConfig
"
,
itemwise
:
true
,
},
],
};
const
procedure
=
new
TranslationProcedure
(
definition
,
{
strict
:
false
});
expect
(
warnSpy
).
toHaveBeenCalledTimes
(
4
);
expect
(
procedure
.
toJSON
()).
toEqual
({
selections
:
[
{
type
:
"
citoid
"
,
config
:
"
itemType
"
,
},
],
transformations
:
[
{
type
:
"
range
"
,
config
:
"
0
"
,
itemwise
:
true
,
},
],
});
expect
(()
=>
{
new
TranslationProcedure
(
definition
);
}).
toThrow
();
});
// empty selection output should give empty transformation output
src/templates/procedure.ts
View file @
4072efda
...
...
@@ -3,6 +3,7 @@ import { Transformation } from "./transformation";
import
{
Webpage
}
from
"
../webpage/webpage
"
;
import
{
StepOutput
}
from
"
../types
"
;
import
{
ProcedureDefinition
,
ProcedureOutput
}
from
"
../types
"
;
import
log
from
"
loglevel
"
;
export
class
TranslationProcedure
{
selections
:
Array
<
Selection
>
;
...
...
@@ -12,13 +13,48 @@ export class TranslationProcedure {
procedure
:
ProcedureDefinition
=
{
selections
:
[],
transformations
:
[],
}
},
{
strict
=
true
,
}:
{
strict
?:
boolean
;
}
=
{}
)
{
this
.
selections
=
procedure
.
selections
.
map
((
selection
)
=>
Selection
.
create
(
selection
)
this
.
selections
=
procedure
.
selections
.
reduce
(
(
selections
:
Selection
[],
selection
)
=>
{
try
{
selections
.
push
(
Selection
.
create
(
selection
));
}
catch
(
e
)
{
if
(
!
strict
)
{
const
type
=
selection
.
type
??
"
untitled
"
;
log
.
warn
(
`Failed to parse "
${
type
}
" selection step definition:
${
e
}
`
);
}
else
{
throw
e
;
}
}
return
selections
;
},
[]
);
this
.
transformations
=
procedure
.
transformations
.
map
((
transformation
)
=>
Transformation
.
create
(
transformation
)
this
.
transformations
=
procedure
.
transformations
.
reduce
(
(
transformations
:
Transformation
[],
transformation
)
=>
{
try
{
transformations
.
push
(
Transformation
.
create
(
transformation
));
}
catch
(
e
)
{
if
(
!
strict
)
{
const
type
=
transformation
.
type
??
"
untitled
"
;
log
.
warn
(
`Failed to parse "
${
type
}
" transformation step definition:
${
e
}
`
);
}
else
{
throw
e
;
}
}
return
transformations
;
},
[]
);
}
...
...
src/templates/template.test.ts
View file @
4072efda
...
...
@@ -4,6 +4,7 @@ import { Webpage } from "../webpage/webpage";
import
*
as
nodeFetch
from
"
node-fetch
"
;
import
{
pages
}
from
"
../webpage/samplePages
"
;
import
{
TemplateFieldDefinition
,
TemplateDefinition
}
from
"
../types
"
;
import
log
from
"
loglevel
"
;
const
mockNodeFetch
=
nodeFetch
as
typeof
import
(
"
../../__mocks__/node-fetch
"
);
...
...
@@ -90,7 +91,7 @@ it("outputs a JSON template definition", () => {
fields
:
[],
});
template
.
label
=
"
sample label
"
;
template
.
addField
(
new
TemplateField
(
"
itemType
"
,
true
));
template
.
addField
(
new
TemplateField
(
"
itemType
"
,
{
loadDefaults
:
true
}
));
expect
(
template
.
toJSON
()).
toEqual
<
TemplateDefinition
>
({
path
:
"
/article1
"
,
label
:
"
sample label
"
,
...
...
@@ -113,3 +114,60 @@ it("outputs a JSON template definition", () => {
],
});
});
it
(
"
constructor optionally skips invalid field definitions
"
,
()
=>
{
const
warnSpy
=
jest
.
spyOn
(
log
,
"
warn
"
).
mockImplementation
();
const
definition
:
unknown
=
{
path
:
"
/
"
,
fields
:
[
{
fieldname
:
"
itemType
"
,
required
:
true
,
procedures
:
[],
},
{
fieldname
:
"
invalidField
"
,
required
:
true
,
procedures
:
[],
},
],
};
const
template
=
new
TranslationTemplate
(
"
example.com
"
,
definition
as
TemplateDefinition
,
{
strict
:
false
}
);
expect
(
warnSpy
).
toHaveBeenCalledTimes
(
1
);
expect
(
template
.
toJSON
()).
toEqual
({
path
:
"
/
"
,
label
:
""
,
fields
:
[
{
fieldname
:
"
itemType
"
,
required
:
true
,
procedures
:
[],
},
],
});
expect
(()
=>
{
new
TranslationTemplate
(
"
example.com
"
,
definition
as
TemplateDefinition
);
}).
toThrow
();
});
it
(
"
rejects creation if a mandatory field is missing
"
,
()
=>
{
const
definition
:
TemplateDefinition
=
{
path
:
"
/
"
,
fields
:
[
{
fieldname
:
"
title
"
,
required
:
true
,
procedures
:
[],
},
],
};
expect
(()
=>
{
new
TranslationTemplate
(
"
example.com
"
,
definition
,
{
forceRequiredFields
:
[
"
itemType
"
,
"
title
"
],
});
}).
toThrow
(
'
Mandatory field "itemType" missing from template definition
'
);
});
src/templates/template.ts
View file @
4072efda
...
...
@@ -18,26 +18,58 @@ export abstract class BaseTranslationTemplate {
protected
constructor
(
domain
:
string
,
template
:
TemplateDefinition
|
FallbackTemplateDefinition
,
forceRequiredFields
:
Array
<
FieldName
>
=
[]
{
forceRequiredFields
=
[],
strict
=
true
,
}:
{
forceRequiredFields
?:
Array
<
FieldName
>
;
strict
?:
boolean
;
}
=
{}
)
{
if
(
!
isDomainName
(
domain
))
{
throw
new
DomainNameError
(
domain
);
}
// reject template creation if any mandatory field is missing
if
(
"
fields
"
in
template
)
{
const
fieldnames
=
template
.
fields
.
map
((
field
)
=>
field
.
fieldname
);
for
(
const
field
of
forceRequiredFields
)
{
if
(
!
fieldnames
.
includes
(
field
))
{
throw
new
MissingFieldError
(
field
);
}
}
}
this
.
domain
=
domain
;
this
.
forceRequiredFields
=
forceRequiredFields
;
this
.
label
=
template
.
label
??
""
;
if
(
template
.
fields
)
{
template
.
fields
.
forEach
((
definition
)
=>
{
const
field
=
new
TemplateField
(
definition
)
;
let
field
;
try
{
this
.
addField
(
field
);
field
=
new
TemplateField
(
definition
,
{
strict
}
);
}
catch
(
e
)
{
if
(
e
instanceof
DuplicateFieldError
)
{
log
.
info
(
`Skipping duplicate field "
${
field
.
name
}
"`
);
if
(
!
strict
)
{
const
fieldname
=
definition
.
fieldname
??
"
untitled
"
;
log
.
warn
(
`Failed to parse "
${
fieldname
}
" template field definition:
${
e
}
`
);
}
else
{
throw
e
;
}
}
if
(
field
!==
undefined
)
{
try
{
this
.
addField
(
field
);
}
catch
(
e
)
{
if
(
e
instanceof
DuplicateFieldError
)
{
log
.
info
(
`Skipping duplicate field "
${
field
.
name
}
"`
);
}
else
{
log
.
info
(
``
);
throw
e
;
}
}
}
});
}
}
...
...
@@ -99,9 +131,15 @@ export class TranslationTemplate extends BaseTranslationTemplate {
constructor
(
domain
:
string
,
template
:
TemplateDefinition
,
forceRequiredFields
:
Array
<
FieldName
>
=
[]
{
forceRequiredFields
=
[],
strict
=
true
,
}:
{
forceRequiredFields
?:
Array
<
FieldName
>
;
strict
?:
boolean
;
}
=
{}
)
{
super
(
domain
,
template
,
forceRequiredFields
);