Update README.md
Browse files
README.md
CHANGED
|
@@ -7,661 +7,6 @@ tags:
|
|
| 7 |
- dataset_size:901028
|
| 8 |
- loss:CosineSimilarityLoss
|
| 9 |
base_model: Shuu12121/CodeModernBERT-Owl
|
| 10 |
-
widget:
|
| 11 |
-
- source_sentence: |2
|
| 12 |
-
public void scan() throws Throwable {
|
| 13 |
-
client = new FTPClient();
|
| 14 |
-
log.info("connecting to " + host + "...");
|
| 15 |
-
client.connect(host);
|
| 16 |
-
log.info(client.getReplyString());
|
| 17 |
-
log.info("logging in...");
|
| 18 |
-
client.login("anonymous", "");
|
| 19 |
-
log.info(client.getReplyString());
|
| 20 |
-
Date date = Calendar.getInstance().getTime();
|
| 21 |
-
xmlDocument = new XMLDocument(host, dir, date);
|
| 22 |
-
scanDirectory(dir);
|
| 23 |
-
}
|
| 24 |
-
sentences:
|
| 25 |
-
- |2
|
| 26 |
-
public static void zip(ZipOutputStream out, File f, String base) throws Exception {
|
| 27 |
-
if (f.isDirectory()) {
|
| 28 |
-
File[] fl = f.listFiles();
|
| 29 |
-
base = base.length() == 0 ? "" : base + File.separator;
|
| 30 |
-
for (int i = 0; i < fl.length; i++) {
|
| 31 |
-
zip(out, fl[i], base + fl[i].getName());
|
| 32 |
-
}
|
| 33 |
-
} else {
|
| 34 |
-
out.putNextEntry(new org.apache.tools.zip.ZipEntry(base));
|
| 35 |
-
FileInputStream in = new FileInputStream(f);
|
| 36 |
-
IOUtils.copyStream(in, out);
|
| 37 |
-
in.close();
|
| 38 |
-
}
|
| 39 |
-
Thread.sleep(10);
|
| 40 |
-
}
|
| 41 |
-
- |2
|
| 42 |
-
public static void zip(String destination, String folder) {
|
| 43 |
-
File fdir = new File(folder);
|
| 44 |
-
File[] files = fdir.listFiles();
|
| 45 |
-
PrintWriter stdout = new PrintWriter(System.out, true);
|
| 46 |
-
int read = 0;
|
| 47 |
-
FileInputStream in;
|
| 48 |
-
byte[] data = new byte[1024];
|
| 49 |
-
try {
|
| 50 |
-
ZipOutputStream out = new ZipOutputStream(new FileOutputStream(destination));
|
| 51 |
-
out.setMethod(ZipOutputStream.DEFLATED);
|
| 52 |
-
for (int i = 0; i < files.length; i++) {
|
| 53 |
-
try {
|
| 54 |
-
stdout.println(files[i].getName());
|
| 55 |
-
ZipEntry entry = new ZipEntry(files[i].getName());
|
| 56 |
-
in = new FileInputStream(files[i].getPath());
|
| 57 |
-
out.putNextEntry(entry);
|
| 58 |
-
while ((read = in.read(data, 0, 1024)) != -1) {
|
| 59 |
-
out.write(data, 0, read);
|
| 60 |
-
}
|
| 61 |
-
out.closeEntry();
|
| 62 |
-
in.close();
|
| 63 |
-
} catch (Exception e) {
|
| 64 |
-
e.printStackTrace();
|
| 65 |
-
}
|
| 66 |
-
}
|
| 67 |
-
out.close();
|
| 68 |
-
} catch (IOException ex) {
|
| 69 |
-
ex.printStackTrace();
|
| 70 |
-
}
|
| 71 |
-
}
|
| 72 |
-
- |2
|
| 73 |
-
private void copyFile(URL from, File to) {
|
| 74 |
-
try {
|
| 75 |
-
InputStream is = from.openStream();
|
| 76 |
-
IOUtils.copy(is, new FileOutputStream(to));
|
| 77 |
-
} catch (IOException e) {
|
| 78 |
-
e.printStackTrace();
|
| 79 |
-
}
|
| 80 |
-
}
|
| 81 |
-
- source_sentence: |2
|
| 82 |
-
public void createMd5Hash() {
|
| 83 |
-
try {
|
| 84 |
-
String vcardObject = new ContactToVcard(TimeZone.getTimeZone("UTC"), "UTF-8").convert(this);
|
| 85 |
-
MessageDigest m = MessageDigest.getInstance("MD5");
|
| 86 |
-
m.update(vcardObject.getBytes());
|
| 87 |
-
this.md5Hash = new BigInteger(m.digest()).toString();
|
| 88 |
-
if (log.isTraceEnabled()) {
|
| 89 |
-
log.trace("Hash is:" + this.md5Hash);
|
| 90 |
-
}
|
| 91 |
-
} catch (ConverterException ex) {
|
| 92 |
-
log.error("Error creating hash:" + ex.getMessage());
|
| 93 |
-
} catch (NoSuchAlgorithmException noSuchAlgorithmException) {
|
| 94 |
-
log.error("Error creating hash:" + noSuchAlgorithmException.getMessage());
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
sentences:
|
| 98 |
-
- |2
|
| 99 |
-
public static void main(String[] args) {
|
| 100 |
-
try {
|
| 101 |
-
int encodeFlag = 0;
|
| 102 |
-
if (args[0].equals("-e")) {
|
| 103 |
-
encodeFlag = Base64.ENCODE;
|
| 104 |
-
} else if (args[0].equals("-d")) {
|
| 105 |
-
encodeFlag = Base64.DECODE;
|
| 106 |
-
}
|
| 107 |
-
String infile = args[1];
|
| 108 |
-
String outfile = args[2];
|
| 109 |
-
File fin = new File(infile);
|
| 110 |
-
FileInputStream fis = new FileInputStream(fin);
|
| 111 |
-
BufferedInputStream bis = new BufferedInputStream(fis);
|
| 112 |
-
Base64.InputStream b64in = new Base64.InputStream(bis, encodeFlag | Base64.DO_BREAK_LINES);
|
| 113 |
-
File fout = new File(outfile);
|
| 114 |
-
FileOutputStream fos = new FileOutputStream(fout);
|
| 115 |
-
BufferedOutputStream bos = new BufferedOutputStream(fos);
|
| 116 |
-
byte[] buff = new byte[1024];
|
| 117 |
-
int read = -1;
|
| 118 |
-
while ((read = b64in.read(buff)) >= 0) {
|
| 119 |
-
bos.write(buff, 0, read);
|
| 120 |
-
}
|
| 121 |
-
bos.close();
|
| 122 |
-
b64in.close();
|
| 123 |
-
} catch (Exception e) {
|
| 124 |
-
e.printStackTrace();
|
| 125 |
-
}
|
| 126 |
-
}
|
| 127 |
-
- |2
|
| 128 |
-
public static String md5(String str) {
|
| 129 |
-
StringBuffer buf = new StringBuffer();
|
| 130 |
-
try {
|
| 131 |
-
MessageDigest md = MessageDigest.getInstance("MD5");
|
| 132 |
-
byte[] data = new byte[32];
|
| 133 |
-
md.update(str.getBytes(md5Encoding), 0, str.length());
|
| 134 |
-
data = md.digest();
|
| 135 |
-
for (int i = 0; i < data.length; i++) {
|
| 136 |
-
int halfbyte = (data[i] >>> 4) & 0x0F;
|
| 137 |
-
int two_halfs = 0;
|
| 138 |
-
do {
|
| 139 |
-
if ((0 <= halfbyte) && (halfbyte <= 9)) buf.append((char) ('0' + halfbyte)); else buf.append((char) ('a' + (halfbyte - 10)));
|
| 140 |
-
halfbyte = data[i] & 0x0F;
|
| 141 |
-
} while (two_halfs++ < 1);
|
| 142 |
-
}
|
| 143 |
-
} catch (Exception e) {
|
| 144 |
-
errorLog("{Malgn.md5} " + e.getMessage());
|
| 145 |
-
}
|
| 146 |
-
return buf.toString();
|
| 147 |
-
}
|
| 148 |
-
- |2
|
| 149 |
-
public static String md5(String text, String charset) {
|
| 150 |
-
MessageDigest msgDigest = null;
|
| 151 |
-
try {
|
| 152 |
-
msgDigest = MessageDigest.getInstance("MD5");
|
| 153 |
-
} catch (NoSuchAlgorithmException e) {
|
| 154 |
-
throw new IllegalStateException("System doesn't support MD5 algorithm.");
|
| 155 |
-
}
|
| 156 |
-
msgDigest.update(text.getBytes());
|
| 157 |
-
byte[] bytes = msgDigest.digest();
|
| 158 |
-
byte tb;
|
| 159 |
-
char low;
|
| 160 |
-
char high;
|
| 161 |
-
char tmpChar;
|
| 162 |
-
String md5Str = new String();
|
| 163 |
-
for (int i = 0; i < bytes.length; i++) {
|
| 164 |
-
tb = bytes[i];
|
| 165 |
-
tmpChar = (char) ((tb >>> 4) & 0x000f);
|
| 166 |
-
if (tmpChar >= 10) {
|
| 167 |
-
high = (char) (('a' + tmpChar) - 10);
|
| 168 |
-
} else {
|
| 169 |
-
high = (char) ('0' + tmpChar);
|
| 170 |
-
}
|
| 171 |
-
md5Str += high;
|
| 172 |
-
tmpChar = (char) (tb & 0x000f);
|
| 173 |
-
if (tmpChar >= 10) {
|
| 174 |
-
low = (char) (('a' + tmpChar) - 10);
|
| 175 |
-
} else {
|
| 176 |
-
low = (char) ('0' + tmpChar);
|
| 177 |
-
}
|
| 178 |
-
md5Str += low;
|
| 179 |
-
}
|
| 180 |
-
return md5Str;
|
| 181 |
-
}
|
| 182 |
-
- source_sentence: |2
|
| 183 |
-
@Override
|
| 184 |
-
public void run() {
|
| 185 |
-
try {
|
| 186 |
-
IOUtils.copy(getSource(), processStdIn);
|
| 187 |
-
System.err.println("Copy done.");
|
| 188 |
-
close();
|
| 189 |
-
} catch (IOException e) {
|
| 190 |
-
e.printStackTrace();
|
| 191 |
-
IOUtils.closeQuietly(ExternalDecoder.this);
|
| 192 |
-
}
|
| 193 |
-
}
|
| 194 |
-
sentences:
|
| 195 |
-
- |2
|
| 196 |
-
private String readJsonString() {
|
| 197 |
-
StringBuilder builder = new StringBuilder();
|
| 198 |
-
HttpClient client = new DefaultHttpClient();
|
| 199 |
-
HttpGet httpGet = new HttpGet(SERVER_URL);
|
| 200 |
-
try {
|
| 201 |
-
HttpResponse response = client.execute(httpGet);
|
| 202 |
-
StatusLine statusLine = response.getStatusLine();
|
| 203 |
-
int statusCode = statusLine.getStatusCode();
|
| 204 |
-
if (statusCode == 200) {
|
| 205 |
-
HttpEntity entity = response.getEntity();
|
| 206 |
-
InputStream content = entity.getContent();
|
| 207 |
-
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
|
| 208 |
-
String line;
|
| 209 |
-
while ((line = reader.readLine()) != null) {
|
| 210 |
-
builder.append(line);
|
| 211 |
-
}
|
| 212 |
-
} else {
|
| 213 |
-
Log.e(TAG, "Failed to download file");
|
| 214 |
-
}
|
| 215 |
-
} catch (ClientProtocolException e) {
|
| 216 |
-
e.printStackTrace();
|
| 217 |
-
} catch (IOException e) {
|
| 218 |
-
e.printStackTrace();
|
| 219 |
-
}
|
| 220 |
-
return builder.toString();
|
| 221 |
-
}
|
| 222 |
-
- |2
|
| 223 |
-
public void run() {
|
| 224 |
-
LogPrinter.log(Level.FINEST, "Started Download at : {0, date, long}", new Date());
|
| 225 |
-
if (!PipeConnected) {
|
| 226 |
-
throw new IllegalStateException("You should connect the pipe before with getInputStream()");
|
| 227 |
-
}
|
| 228 |
-
InputStream ins = null;
|
| 229 |
-
if (IsAlreadyDownloaded) {
|
| 230 |
-
LogPrinter.log(Level.FINEST, "The file already Exists open and foward the byte");
|
| 231 |
-
try {
|
| 232 |
-
ContentLength = (int) TheAskedFile.length();
|
| 233 |
-
ContentType = URLConnection.getFileNameMap().getContentTypeFor(TheAskedFile.getName());
|
| 234 |
-
ins = new FileInputStream(TheAskedFile);
|
| 235 |
-
byte[] buffer = new byte[BUFFER_SIZE];
|
| 236 |
-
int read = ins.read(buffer);
|
| 237 |
-
while (read >= 0) {
|
| 238 |
-
Pipe.write(buffer, 0, read);
|
| 239 |
-
read = ins.read(buffer);
|
| 240 |
-
}
|
| 241 |
-
} catch (IOException e) {
|
| 242 |
-
e.printStackTrace();
|
| 243 |
-
} finally {
|
| 244 |
-
if (ins != null) {
|
| 245 |
-
try {
|
| 246 |
-
ins.close();
|
| 247 |
-
} catch (IOException e) {
|
| 248 |
-
}
|
| 249 |
-
}
|
| 250 |
-
}
|
| 251 |
-
} else {
|
| 252 |
-
LogPrinter.log(Level.FINEST, "the file does not exist locally so we try to download the thing");
|
| 253 |
-
File theDir = TheAskedFile.getParentFile();
|
| 254 |
-
if (!theDir.exists()) {
|
| 255 |
-
theDir.mkdirs();
|
| 256 |
-
}
|
| 257 |
-
for (URL url : ListFastest) {
|
| 258 |
-
FileOutputStream fout = null;
|
| 259 |
-
boolean OnError = false;
|
| 260 |
-
long timestart = System.currentTimeMillis();
|
| 261 |
-
long bytecount = 0;
|
| 262 |
-
try {
|
| 263 |
-
URL newUrl = new URL(url.toString() + RequestedFile);
|
| 264 |
-
LogPrinter.log(Level.FINEST, "the download URL = {0}", newUrl);
|
| 265 |
-
URLConnection conn = newUrl.openConnection();
|
| 266 |
-
ContentType = conn.getContentType();
|
| 267 |
-
ContentLength = conn.getContentLength();
|
| 268 |
-
ins = conn.getInputStream();
|
| 269 |
-
fout = new FileOutputStream(TheAskedFile);
|
| 270 |
-
byte[] buffer = new byte[BUFFER_SIZE];
|
| 271 |
-
int read = ins.read(buffer);
|
| 272 |
-
while (read >= 0) {
|
| 273 |
-
fout.write(buffer, 0, read);
|
| 274 |
-
Pipe.write(buffer, 0, read);
|
| 275 |
-
read = ins.read(buffer);
|
| 276 |
-
bytecount += read;
|
| 277 |
-
}
|
| 278 |
-
Pipe.flush();
|
| 279 |
-
} catch (IOException e) {
|
| 280 |
-
OnError = true;
|
| 281 |
-
} finally {
|
| 282 |
-
if (ins != null) {
|
| 283 |
-
try {
|
| 284 |
-
ins.close();
|
| 285 |
-
} catch (IOException e) {
|
| 286 |
-
}
|
| 287 |
-
}
|
| 288 |
-
if (fout != null) {
|
| 289 |
-
try {
|
| 290 |
-
fout.close();
|
| 291 |
-
} catch (IOException e) {
|
| 292 |
-
}
|
| 293 |
-
}
|
| 294 |
-
}
|
| 295 |
-
long timeend = System.currentTimeMillis();
|
| 296 |
-
if (OnError) {
|
| 297 |
-
continue;
|
| 298 |
-
} else {
|
| 299 |
-
long timetook = timeend - timestart;
|
| 300 |
-
BigDecimal speed = new BigDecimal(bytecount).multiply(new BigDecimal(1000)).divide(new BigDecimal(timetook), MathContext.DECIMAL32);
|
| 301 |
-
for (ReportCalculatedStatistique report : Listener) {
|
| 302 |
-
report.reportUrlStat(url, speed, timetook);
|
| 303 |
-
}
|
| 304 |
-
break;
|
| 305 |
-
}
|
| 306 |
-
}
|
| 307 |
-
}
|
| 308 |
-
LogPrinter.log(Level.FINEST, "download finished at {0,date,long}", new Date());
|
| 309 |
-
if (Pipe != null) {
|
| 310 |
-
try {
|
| 311 |
-
Pipe.close();
|
| 312 |
-
} catch (IOException e) {
|
| 313 |
-
e.printStackTrace();
|
| 314 |
-
}
|
| 315 |
-
}
|
| 316 |
-
}
|
| 317 |
-
- |2
|
| 318 |
-
public void run(String srcf, String dst) {
|
| 319 |
-
final Path srcPath = new Path("./" + srcf);
|
| 320 |
-
final Path desPath = new Path(dst);
|
| 321 |
-
try {
|
| 322 |
-
Path[] srcs = FileUtil.stat2Paths(hdfs.globStatus(srcPath), srcPath);
|
| 323 |
-
OutputStream out = FileSystem.getLocal(conf).create(desPath);
|
| 324 |
-
for (int i = 0; i < srcs.length; i++) {
|
| 325 |
-
System.out.println(srcs[i]);
|
| 326 |
-
InputStream in = hdfs.open(srcs[i]);
|
| 327 |
-
IOUtils.copyBytes(in, out, conf, false);
|
| 328 |
-
in.close();
|
| 329 |
-
}
|
| 330 |
-
out.close();
|
| 331 |
-
} catch (IOException ex) {
|
| 332 |
-
System.err.print(ex.getMessage());
|
| 333 |
-
}
|
| 334 |
-
}
|
| 335 |
-
- source_sentence: |2
|
| 336 |
-
private void readHomePage(ITestThread testThread) throws IOException {
|
| 337 |
-
if (null == testThread) {
|
| 338 |
-
throw new IllegalArgumentException("Test thread may not be null.");
|
| 339 |
-
}
|
| 340 |
-
final InputStream urlIn = new URL(testUrl).openStream();
|
| 341 |
-
final int availableBytes = urlIn.available();
|
| 342 |
-
if (0 == availableBytes) {
|
| 343 |
-
throw new IllegalStateException("Zero bytes on target host.");
|
| 344 |
-
}
|
| 345 |
-
in = new BufferedReader(new InputStreamReader(urlIn));
|
| 346 |
-
String line;
|
| 347 |
-
while (null != in && null != (line = in.readLine())) {
|
| 348 |
-
page.append(line);
|
| 349 |
-
page.append('\n');
|
| 350 |
-
if (0 != lineDelay) {
|
| 351 |
-
OS.sleep(lineDelay);
|
| 352 |
-
}
|
| 353 |
-
if (testThread.isActionStopped()) {
|
| 354 |
-
break;
|
| 355 |
-
}
|
| 356 |
-
}
|
| 357 |
-
}
|
| 358 |
-
sentences:
|
| 359 |
-
- |2
|
| 360 |
-
@Override
|
| 361 |
-
public ReturnValue do_run() {
|
| 362 |
-
int bufLen = 500 * 1024;
|
| 363 |
-
ReturnValue ret = new ReturnValue();
|
| 364 |
-
ret.setExitStatus(ReturnValue.SUCCESS);
|
| 365 |
-
File output = null;
|
| 366 |
-
if (((String) options.valueOf("input-file")).startsWith("s3://")) {
|
| 367 |
-
Pattern p = Pattern.compile("s3://(\\S+):(\\S+)@(\\S+)");
|
| 368 |
-
Matcher m = p.matcher((String) options.valueOf("input-file"));
|
| 369 |
-
boolean result = m.find();
|
| 370 |
-
String accessKey = null;
|
| 371 |
-
String secretKey = null;
|
| 372 |
-
String URL = (String) options.valueOf("input-file");
|
| 373 |
-
if (result) {
|
| 374 |
-
accessKey = m.group(1);
|
| 375 |
-
secretKey = m.group(2);
|
| 376 |
-
URL = "s3://" + m.group(3);
|
| 377 |
-
} else {
|
| 378 |
-
try {
|
| 379 |
-
HashMap<String, String> settings = (HashMap<String, String>) ConfigTools.getSettings();
|
| 380 |
-
accessKey = settings.get("AWS_ACCESS_KEY");
|
| 381 |
-
secretKey = settings.get("AWS_SECRET_KEY");
|
| 382 |
-
} catch (Exception e) {
|
| 383 |
-
ret.setExitStatus(ReturnValue.SETTINGSFILENOTFOUND);
|
| 384 |
-
ret.setProcessExitStatus(ReturnValue.SETTINGSFILENOTFOUND);
|
| 385 |
-
return (ret);
|
| 386 |
-
}
|
| 387 |
-
}
|
| 388 |
-
if (accessKey == null || secretKey == null) {
|
| 389 |
-
ret.setExitStatus(ReturnValue.ENVVARNOTFOUND);
|
| 390 |
-
ret.setProcessExitStatus(ReturnValue.ENVVARNOTFOUND);
|
| 391 |
-
return (ret);
|
| 392 |
-
}
|
| 393 |
-
AmazonS3 s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
|
| 394 |
-
p = Pattern.compile("s3://([^/]+)/(\\S+)");
|
| 395 |
-
m = p.matcher(URL);
|
| 396 |
-
result = m.find();
|
| 397 |
-
if (result) {
|
| 398 |
-
String bucket = m.group(1);
|
| 399 |
-
String key = m.group(2);
|
| 400 |
-
S3Object object = s3.getObject(new GetObjectRequest(bucket, key));
|
| 401 |
-
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
|
| 402 |
-
output = new File((String) options.valueOf("output-dir") + File.separator + key);
|
| 403 |
-
output.getParentFile().mkdirs();
|
| 404 |
-
if (!output.exists() || output.length() != object.getObjectMetadata().getContentLength()) {
|
| 405 |
-
System.out.println("Downloading an S3 object from bucket: " + bucket + " with key: " + key);
|
| 406 |
-
BufferedInputStream reader = new BufferedInputStream(object.getObjectContent(), bufLen);
|
| 407 |
-
try {
|
| 408 |
-
BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(output), bufLen);
|
| 409 |
-
while (true) {
|
| 410 |
-
int data = reader.read();
|
| 411 |
-
if (data == -1) {
|
| 412 |
-
break;
|
| 413 |
-
}
|
| 414 |
-
writer.write(data);
|
| 415 |
-
}
|
| 416 |
-
reader.close();
|
| 417 |
-
writer.close();
|
| 418 |
-
} catch (FileNotFoundException e) {
|
| 419 |
-
System.err.println(e.getMessage());
|
| 420 |
-
} catch (IOException e) {
|
| 421 |
-
System.err.println(e.getMessage());
|
| 422 |
-
}
|
| 423 |
-
} else {
|
| 424 |
-
System.out.println("Skipping download of S3 object from bucket: " + bucket + " with key: " + key + " since local output exists: " + output.getAbsolutePath());
|
| 425 |
-
}
|
| 426 |
-
}
|
| 427 |
-
} else if (((String) options.valueOf("input-file")).startsWith("http://") || ((String) options.valueOf("input-file")).startsWith("https://")) {
|
| 428 |
-
Pattern p = Pattern.compile("(https*)://(\\S+):(\\S+)@(\\S+)");
|
| 429 |
-
Matcher m = p.matcher((String) options.valueOf("input-file"));
|
| 430 |
-
boolean result = m.find();
|
| 431 |
-
String protocol = null;
|
| 432 |
-
String user = null;
|
| 433 |
-
String pass = null;
|
| 434 |
-
String URL = (String) options.valueOf("input-file");
|
| 435 |
-
if (result) {
|
| 436 |
-
protocol = m.group(1);
|
| 437 |
-
user = m.group(2);
|
| 438 |
-
pass = m.group(3);
|
| 439 |
-
URL = protocol + "://" + m.group(4);
|
| 440 |
-
}
|
| 441 |
-
URL urlObj = null;
|
| 442 |
-
try {
|
| 443 |
-
urlObj = new URL(URL);
|
| 444 |
-
if (urlObj != null) {
|
| 445 |
-
URLConnection urlConn = urlObj.openConnection();
|
| 446 |
-
if (user != null && pass != null) {
|
| 447 |
-
String userPassword = user + ":" + pass;
|
| 448 |
-
String encoding = new Base64().encodeBase64String(userPassword.getBytes());
|
| 449 |
-
urlConn.setRequestProperty("Authorization", "Basic " + encoding);
|
| 450 |
-
}
|
| 451 |
-
p = Pattern.compile("://([^/]+)/(\\S+)");
|
| 452 |
-
m = p.matcher(URL);
|
| 453 |
-
result = m.find();
|
| 454 |
-
if (result) {
|
| 455 |
-
String host = m.group(1);
|
| 456 |
-
String path = m.group(2);
|
| 457 |
-
output = new File((String) options.valueOf("output-dir") + path);
|
| 458 |
-
output.getParentFile().mkdirs();
|
| 459 |
-
if (!output.exists() || output.length() != urlConn.getContentLength()) {
|
| 460 |
-
System.out.println("Downloading an http object from URL: " + URL);
|
| 461 |
-
BufferedInputStream reader = new BufferedInputStream(urlConn.getInputStream(), bufLen);
|
| 462 |
-
BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(output), bufLen);
|
| 463 |
-
while (true) {
|
| 464 |
-
int data = reader.read();
|
| 465 |
-
if (data == -1) {
|
| 466 |
-
break;
|
| 467 |
-
}
|
| 468 |
-
writer.write(data);
|
| 469 |
-
}
|
| 470 |
-
reader.close();
|
| 471 |
-
writer.close();
|
| 472 |
-
} else {
|
| 473 |
-
System.out.println("Skipping download of http object from URL: " + URL + " since local output exists: " + output.getAbsolutePath());
|
| 474 |
-
}
|
| 475 |
-
}
|
| 476 |
-
}
|
| 477 |
-
} catch (MalformedURLException e) {
|
| 478 |
-
System.err.println(e.getMessage());
|
| 479 |
-
} catch (IOException e) {
|
| 480 |
-
System.err.println(e.getMessage());
|
| 481 |
-
}
|
| 482 |
-
} else {
|
| 483 |
-
output = new File((String) options.valueOf("input-file"));
|
| 484 |
-
}
|
| 485 |
-
boolean result = FileTools.unzipFile(output, new File((String) options.valueOf("output-dir")));
|
| 486 |
-
if (!result) {
|
| 487 |
-
ret.setStderr("Can't unzip software bundle " + options.valueOf("input-file") + " to directory " + options.valueOf("output-dir"));
|
| 488 |
-
ret.setExitStatus(ReturnValue.RUNTIMEEXCEPTION);
|
| 489 |
-
}
|
| 490 |
-
return (ret);
|
| 491 |
-
}
|
| 492 |
-
- |2
|
| 493 |
-
public void writeConfigurationFile() throws IOException, ComponentException {
|
| 494 |
-
SystemConfig config = parent.getParentSystem().getConfiguration();
|
| 495 |
-
File original = config.getLocation();
|
| 496 |
-
File backup = new File(original.getParentFile(), original.getName() + "." + System.currentTimeMillis());
|
| 497 |
-
FileInputStream in = new FileInputStream(original);
|
| 498 |
-
FileOutputStream out = new FileOutputStream(backup);
|
| 499 |
-
byte[] buffer = new byte[2048];
|
| 500 |
-
try {
|
| 501 |
-
int bytesread = 0;
|
| 502 |
-
while ((bytesread = in.read(buffer)) > 0) {
|
| 503 |
-
out.write(buffer, 0, bytesread);
|
| 504 |
-
}
|
| 505 |
-
} catch (IOException e) {
|
| 506 |
-
logger.warn("Failed to copy backup of configuration file");
|
| 507 |
-
throw e;
|
| 508 |
-
} finally {
|
| 509 |
-
in.close();
|
| 510 |
-
out.close();
|
| 511 |
-
}
|
| 512 |
-
FileWriter replace = new FileWriter(original);
|
| 513 |
-
replace.write(config.toFileFormat());
|
| 514 |
-
replace.close();
|
| 515 |
-
logger.info("Re-wrote configuration file " + original.getPath());
|
| 516 |
-
}
|
| 517 |
-
- |2
|
| 518 |
-
@Override
|
| 519 |
-
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
|
| 520 |
-
resp.addHeader("Cache-Control", "max-age=" + Constants.HTTP_CACHE_SECONDS);
|
| 521 |
-
String uuid = req.getRequestURI().substring(req.getRequestURI().indexOf(Constants.SERVLET_FULL_PREFIX) + Constants.SERVLET_FULL_PREFIX.length() + 1);
|
| 522 |
-
boolean notScale = ClientUtils.toBoolean(req.getParameter(Constants.URL_PARAM_NOT_SCALE));
|
| 523 |
-
ServletOutputStream os = resp.getOutputStream();
|
| 524 |
-
if (uuid != null && !"".equals(uuid)) {
|
| 525 |
-
try {
|
| 526 |
-
String mimetype = fedoraAccess.getMimeTypeForStream(uuid, FedoraUtils.IMG_FULL_STREAM);
|
| 527 |
-
if (mimetype == null) {
|
| 528 |
-
mimetype = "image/jpeg";
|
| 529 |
-
}
|
| 530 |
-
ImageMimeType loadFromMimeType = ImageMimeType.loadFromMimeType(mimetype);
|
| 531 |
-
if (loadFromMimeType == ImageMimeType.JPEG || loadFromMimeType == ImageMimeType.PNG) {
|
| 532 |
-
StringBuffer sb = new StringBuffer();
|
| 533 |
-
sb.append(config.getFedoraHost()).append("/objects/").append(uuid).append("/datastreams/IMG_FULL/content");
|
| 534 |
-
InputStream is = RESTHelper.get(sb.toString(), config.getFedoraLogin(), config.getFedoraPassword(), false);
|
| 535 |
-
if (is == null) {
|
| 536 |
-
return;
|
| 537 |
-
}
|
| 538 |
-
try {
|
| 539 |
-
IOUtils.copyStreams(is, os);
|
| 540 |
-
} catch (IOException e) {
|
| 541 |
-
resp.setStatus(HttpURLConnection.HTTP_NOT_FOUND);
|
| 542 |
-
LOGGER.error("Unable to open full image.", e);
|
| 543 |
-
} finally {
|
| 544 |
-
os.flush();
|
| 545 |
-
if (is != null) {
|
| 546 |
-
try {
|
| 547 |
-
is.close();
|
| 548 |
-
} catch (IOException e) {
|
| 549 |
-
resp.setStatus(HttpURLConnection.HTTP_NOT_FOUND);
|
| 550 |
-
LOGGER.error("Unable to close stream.", e);
|
| 551 |
-
} finally {
|
| 552 |
-
is = null;
|
| 553 |
-
}
|
| 554 |
-
}
|
| 555 |
-
}
|
| 556 |
-
} else {
|
| 557 |
-
Image rawImg = KrameriusImageSupport.readImage(uuid, FedoraUtils.IMG_FULL_STREAM, this.fedoraAccess, 0, loadFromMimeType);
|
| 558 |
-
BufferedImage scaled = null;
|
| 559 |
-
if (!notScale) {
|
| 560 |
-
scaled = KrameriusImageSupport.getSmallerImage(rawImg, 1250, 1000);
|
| 561 |
-
} else {
|
| 562 |
-
scaled = KrameriusImageSupport.getSmallerImage(rawImg, 2500, 2000);
|
| 563 |
-
}
|
| 564 |
-
KrameriusImageSupport.writeImageToStream(scaled, "JPG", os);
|
| 565 |
-
resp.setContentType(ImageMimeType.JPEG.getValue());
|
| 566 |
-
resp.setStatus(HttpURLConnection.HTTP_OK);
|
| 567 |
-
}
|
| 568 |
-
} catch (IOException e) {
|
| 569 |
-
resp.setStatus(HttpURLConnection.HTTP_NOT_FOUND);
|
| 570 |
-
LOGGER.error("Unable to open full image.", e);
|
| 571 |
-
} catch (XPathExpressionException e) {
|
| 572 |
-
resp.setStatus(HttpURLConnection.HTTP_NOT_FOUND);
|
| 573 |
-
LOGGER.error("Unable to create XPath expression.", e);
|
| 574 |
-
} finally {
|
| 575 |
-
os.flush();
|
| 576 |
-
}
|
| 577 |
-
}
|
| 578 |
-
}
|
| 579 |
-
- source_sentence: |2
|
| 580 |
-
public static boolean insert(final Cargo cargo) {
|
| 581 |
-
int result = 0;
|
| 582 |
-
final Connection c = DBConnection.getConnection();
|
| 583 |
-
PreparedStatement pst = null;
|
| 584 |
-
if (c == null) {
|
| 585 |
-
return false;
|
| 586 |
-
}
|
| 587 |
-
try {
|
| 588 |
-
c.setAutoCommit(false);
|
| 589 |
-
final String sql = "insert into cargo (nome) values (?)";
|
| 590 |
-
pst = c.prepareStatement(sql);
|
| 591 |
-
pst.setString(1, cargo.getNome());
|
| 592 |
-
result = pst.executeUpdate();
|
| 593 |
-
c.commit();
|
| 594 |
-
} catch (final SQLException e) {
|
| 595 |
-
try {
|
| 596 |
-
c.rollback();
|
| 597 |
-
} catch (final SQLException e1) {
|
| 598 |
-
e1.printStackTrace();
|
| 599 |
-
}
|
| 600 |
-
System.out.println("[CargoDAO.insert] Erro ao inserir -> " + e.getMessage());
|
| 601 |
-
} finally {
|
| 602 |
-
DBConnection.closePreparedStatement(pst);
|
| 603 |
-
DBConnection.closeConnection(c);
|
| 604 |
-
}
|
| 605 |
-
if (result > 0) {
|
| 606 |
-
return true;
|
| 607 |
-
} else {
|
| 608 |
-
return false;
|
| 609 |
-
}
|
| 610 |
-
}
|
| 611 |
-
sentences:
|
| 612 |
-
- |2
|
| 613 |
-
public String md5sum(String toCompute) throws Exception {
|
| 614 |
-
MessageDigest md = MessageDigest.getInstance("MD5");
|
| 615 |
-
md.update(toCompute.getBytes());
|
| 616 |
-
java.math.BigInteger hash = new java.math.BigInteger(1, md.digest());
|
| 617 |
-
return hash.toString(16);
|
| 618 |
-
}
|
| 619 |
-
- |2
|
| 620 |
-
protected void runTest(URL pBaseURL, String pName, String pHref) throws Exception {
|
| 621 |
-
URL url = new URL(pBaseURL, pHref);
|
| 622 |
-
XSParser parser = new XSParser();
|
| 623 |
-
parser.setValidating(false);
|
| 624 |
-
InputSource isource = new InputSource(url.openStream());
|
| 625 |
-
isource.setSystemId(url.toString());
|
| 626 |
-
String result;
|
| 627 |
-
try {
|
| 628 |
-
parser.parse(isource);
|
| 629 |
-
++numOk;
|
| 630 |
-
result = "Ok";
|
| 631 |
-
} catch (Exception e) {
|
| 632 |
-
++numFailed;
|
| 633 |
-
result = e.getMessage();
|
| 634 |
-
}
|
| 635 |
-
log("Running test " + pName + " with URL " + url + ": " + result);
|
| 636 |
-
}
|
| 637 |
-
- |2
|
| 638 |
-
public String generateMappackMD5(File mapPackFile) throws IOException, NoSuchAlgorithmException {
|
| 639 |
-
ZipFile zip = new ZipFile(mapPackFile);
|
| 640 |
-
try {
|
| 641 |
-
Enumeration<? extends ZipEntry> entries = zip.entries();
|
| 642 |
-
MessageDigest md5Total = MessageDigest.getInstance("MD5");
|
| 643 |
-
MessageDigest md5 = MessageDigest.getInstance("MD5");
|
| 644 |
-
while (entries.hasMoreElements()) {
|
| 645 |
-
ZipEntry entry = entries.nextElement();
|
| 646 |
-
if (entry.isDirectory()) continue;
|
| 647 |
-
String name = entry.getName();
|
| 648 |
-
if (name.toUpperCase().startsWith("META-INF")) continue;
|
| 649 |
-
md5.reset();
|
| 650 |
-
InputStream in = zip.getInputStream(entry);
|
| 651 |
-
byte[] data = Utilities.getInputBytes(in);
|
| 652 |
-
in.close();
|
| 653 |
-
byte[] digest = md5.digest(data);
|
| 654 |
-
log.trace("Hashsum " + Hex.encodeHexString(digest) + " includes \"" + name + "\"");
|
| 655 |
-
md5Total.update(digest);
|
| 656 |
-
md5Total.update(name.getBytes());
|
| 657 |
-
}
|
| 658 |
-
String md5sum = Hex.encodeHexString(md5Total.digest());
|
| 659 |
-
log.trace("md5sum of " + mapPackFile.getName() + ": " + md5sum);
|
| 660 |
-
return md5sum;
|
| 661 |
-
} finally {
|
| 662 |
-
zip.close();
|
| 663 |
-
}
|
| 664 |
-
}
|
| 665 |
pipeline_tag: sentence-similarity
|
| 666 |
library_name: sentence-transformers
|
| 667 |
metrics:
|
|
@@ -683,7 +28,7 @@ model-index:
|
|
| 683 |
- type: spearman_cosine
|
| 684 |
value: 0.5635084463158045
|
| 685 |
name: Spearman Cosine
|
| 686 |
-
license:
|
| 687 |
---
|
| 688 |
|
| 689 |
# SentenceTransformer based on Shuu12121/CodeModernBERT-Owl
|
|
|
|
| 7 |
- dataset_size:901028
|
| 8 |
- loss:CosineSimilarityLoss
|
| 9 |
base_model: Shuu12121/CodeModernBERT-Owl
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
pipeline_tag: sentence-similarity
|
| 11 |
library_name: sentence-transformers
|
| 12 |
metrics:
|
|
|
|
| 28 |
- type: spearman_cosine
|
| 29 |
value: 0.5635084463158045
|
| 30 |
name: Spearman Cosine
|
| 31 |
+
license: apache-2.0
|
| 32 |
---
|
| 33 |
|
| 34 |
# SentenceTransformer based on Shuu12121/CodeModernBERT-Owl
|