fix: propper support for both languages, should only be cz and en
This commit is contained in:
+8
-5
@@ -1,10 +1,13 @@
|
|||||||
FROM node:latest
|
FROM node:latest
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY package.json package-lock.json ./
|
||||||
|
|
||||||
|
RUN npm ci
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN npm install -g npm@12.0.2
|
|
||||||
RUN npm install express@5.2.1
|
|
||||||
RUN npm install node-fetch@3.3.2
|
|
||||||
RUN npm install jest@30.5.1
|
|
||||||
RUN npm install supertest@7.2.2
|
|
||||||
CMD ["npm", "run", "serve"]
|
CMD ["npm", "run", "serve"]
|
||||||
|
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
@@ -20,6 +20,12 @@ test-and-run:
|
|||||||
|
|
||||||
deploy: build test rund
|
deploy: build test rund
|
||||||
|
|
||||||
|
reubild:
|
||||||
|
docker compose down
|
||||||
|
docker compose up -d --build
|
||||||
|
|
||||||
|
watch-logs:
|
||||||
|
docker compose logs -f
|
||||||
|
|
||||||
preview:
|
preview:
|
||||||
npm run serve
|
npm run serve
|
||||||
@@ -4,7 +4,8 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "3000:3000"
|
- "3000:3000"
|
||||||
depends_on:
|
depends_on:
|
||||||
- searxng
|
searxng:
|
||||||
|
condition: service_healthy
|
||||||
environment:
|
environment:
|
||||||
SEARXNG_URL: http://searxng:8080
|
SEARXNG_URL: http://searxng:8080
|
||||||
|
|
||||||
@@ -14,3 +15,8 @@ services:
|
|||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
volumes:
|
volumes:
|
||||||
- ./searxng:/etc/searxng
|
- ./searxng:/etc/searxng
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q --spider http://localhost:8080/ || exit 1"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
|||||||
+296
File diff suppressed because it is too large
Load Diff
Generated
+340
-4
File diff suppressed because it is too large
Load Diff
+6
-4
@@ -10,9 +10,11 @@
|
|||||||
"serve": "node src/server.js"
|
"serve": "node src/server.js"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"express": "^5.2.1",
|
"express": "5.2.1",
|
||||||
"jest": "^30.5.1",
|
"node-fetch": "3.3.2"
|
||||||
"node-fetch": "^3.3.2",
|
},
|
||||||
"supertest": "^7.2.2"
|
"devDependencies": {
|
||||||
|
"jest": "30.5.1",
|
||||||
|
"supertest": "7.2.2"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,16 @@ use_default_settings: true
|
|||||||
|
|
||||||
server:
|
server:
|
||||||
bind_address: 0.0.0.0
|
bind_address: 0.0.0.0
|
||||||
port: 3001
|
port: 8080
|
||||||
secret_key: TIUZTTPjIsmMvFhioCihiKPablGHnoQhPSdXBLTfYbGjzoXILBFkTjYxbMULgAAC
|
secret_key: TIUZTTPjIsmMvFhioCihiKPablGHnoQhPSdXBLTfYbGjzoXILBFkTjYxbMULgAAC
|
||||||
|
|
||||||
search:
|
search:
|
||||||
formats:
|
formats:
|
||||||
- html
|
- html
|
||||||
- json
|
- json
|
||||||
|
|
||||||
|
engines:
|
||||||
|
- name: startpage
|
||||||
|
engine: startpage
|
||||||
|
shortcut: sp
|
||||||
|
language: en
|
||||||
+7
-1
@@ -15,6 +15,12 @@ app.get('/en', (req, res) => {
|
|||||||
res.sendFile(path.join(__dirname, '/static/en/index.html'));
|
res.sendFile(path.join(__dirname, '/static/en/index.html'));
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/search', handleSearch);
|
app.get('/search', (req, res) => {
|
||||||
|
handleSearch(req, res, "cz-CS");
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/en/search', (req, res) => {
|
||||||
|
handleSearch(req, res, "en-US");
|
||||||
|
});
|
||||||
|
|
||||||
module.exports = app;
|
module.exports = app;
|
||||||
+46
-42
@@ -1,52 +1,56 @@
|
|||||||
const filterJson = require('./filter-json');
|
const filterJson = require('./filter-json');
|
||||||
|
|
||||||
async function handleSearch(req, res) {
|
async function handleSearch(req, res, lang) {
|
||||||
try {
|
try {
|
||||||
const query = req.query.query;
|
const query = req.query.query;
|
||||||
|
|
||||||
if (!query) {
|
if (!query) {
|
||||||
return res.status(400).send("Missing search query");
|
return res.status(400).send("Missing search query");
|
||||||
}
|
|
||||||
|
|
||||||
/* Here we use a local searx.ng instance (a meta-search engine) and filter out google/startpage
|
|
||||||
* search results. By testing, i've found out that !sp (startpage) ro !goc (google's programmable
|
|
||||||
* search engine) is more reliable for this kind of operation that regular google.
|
|
||||||
* A local instance instead of an already available one is used because of bot detection. :)
|
|
||||||
*/
|
|
||||||
const searxngUrl =
|
|
||||||
process.env.SEARXNG_URL || "http://localhost:8080";
|
|
||||||
|
|
||||||
const params = new URLSearchParams({
|
|
||||||
q: query,
|
|
||||||
engines: "startpage",
|
|
||||||
format: "json"
|
|
||||||
});
|
|
||||||
|
|
||||||
const reposnse = await fetch(
|
|
||||||
`${searxngUrl}/search?${params}`,
|
|
||||||
{
|
|
||||||
headers: {
|
|
||||||
Accept: "application/json"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
|
||||||
const data = await reposnse.json();
|
/* Here we use a local searx.ng instance (a meta-search engine) and filter out google/startpage
|
||||||
const filteredresults = filterJson(data.results || []);
|
* search results. By testing, i've found out that !sp (startpage) ro !goc (google's programmable
|
||||||
|
* search engine) is more reliable for this kind of operation that regular google.
|
||||||
|
* A local instance instead of an already available one is used because of bot detection. :)
|
||||||
|
*/
|
||||||
|
const searxngUrl =
|
||||||
|
process.env.SEARXNG_URL || "http://localhost:8080";
|
||||||
|
|
||||||
// send the result as a downloadable JSON file
|
const params = new URLSearchParams({
|
||||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
q: query,
|
||||||
res.setHeader(
|
engines: "startpage",
|
||||||
"Content-Disposition",
|
format: "json",
|
||||||
'attachment; filename="output.json"'
|
category_general: 1,
|
||||||
);
|
language: lang
|
||||||
|
});
|
||||||
|
console.log(`${searxngUrl}/search?${params}`);
|
||||||
|
|
||||||
return res.send(JSON.stringify(filteredresults, null, 2));
|
const reposnse = await fetch(
|
||||||
} catch (error) {
|
`${searxngUrl}/search?${params}`,
|
||||||
return res
|
{
|
||||||
.status(500)
|
headers: {
|
||||||
.send("<p><b>Search failed!<br>See the console for the error.</b></p>");
|
Accept: "application/json"
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const data = await reposnse.json();
|
||||||
|
const filteredresults = filterJson(data.results || []);
|
||||||
|
|
||||||
|
// send the result as a downloadable JSON file
|
||||||
|
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||||
|
res.setHeader(
|
||||||
|
"Content-Disposition",
|
||||||
|
'attachment; filename="output.json"'
|
||||||
|
);
|
||||||
|
|
||||||
|
return res.send(JSON.stringify(filteredresults, null, 2));
|
||||||
|
} catch (error) {
|
||||||
|
console.log(error)
|
||||||
|
return res
|
||||||
|
.status(500)
|
||||||
|
.send("<p><b>Search failed!<br>See the console for the error.</b></p>");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|||||||
@@ -21,7 +21,7 @@
|
|||||||
</tr>
|
</tr>
|
||||||
</table>
|
</table>
|
||||||
<h3 id="subtitle">Get the search results and save them in a JSON file.</h3>
|
<h3 id="subtitle">Get the search results and save them in a JSON file.</h3>
|
||||||
<form action="/search" name="searchbox" method="GET">
|
<form action="/en/search" name="searchbox" method="GET">
|
||||||
<input id="searchform" placeholder="Type your query here..." type="text" name="query"/>
|
<input id="searchform" placeholder="Type your query here..." type="text" name="query"/>
|
||||||
<br>
|
<br>
|
||||||
<input id="searchbutton" type="submit" value="Search"/>
|
<input id="searchbutton" type="submit" value="Search"/>
|
||||||
|
|||||||
Reference in New Issue
Block a user