libzypp 17.37.1
MediaCurl2.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
12
13#include <iostream>
14
15#include <zypp/base/Logger.h>
17#include <zypp/base/String.h>
18#include <zypp/base/Gettext.h>
19#include <zypp-core/parser/Sysconfig>
20
22
25#include <zypp-curl/ProxyInfo>
26#include <zypp-curl/CurlConfig>
28#include <zypp/Target.h>
29#include <zypp/ZYppFactory.h>
30#include <zypp/ZConfig.h>
31#include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
32
35
36#include <cstdlib>
37#include <sys/types.h>
38#include <sys/stat.h>
39#include <sys/mount.h>
40#include <dirent.h>
41#include <unistd.h>
42#include <glib.h>
43
47
48#ifdef ENABLE_ZCHUNK_COMPRESSION
50#endif
51
52
58
59using std::endl;
60
61using namespace internal;
62using namespace zypp::base;
63
64namespace zypp {
65
66 namespace media {
67
68 MediaCurl2::MediaCurl2(const MediaUrl &url_r, const std::vector<MediaUrl> &mirrors_r,
69 const Pathname & attach_point_hint_r )
70 : MediaNetworkCommonHandler( url_r, mirrors_r, attach_point_hint_r,
71 "/", // urlpath at attachpoint
72 true ) // does_download
73 , _executor( std::make_shared<internal::MediaNetworkRequestExecutor>() )
74 {
75
76 MIL << "MediaCurl2::MediaCurl2(" << url_r.url() << ", " << attach_point_hint_r << ")" << endl;
77
78 if( !attachPoint().empty())
79 {
80 PathInfo ainfo(attachPoint());
81 Pathname apath(attachPoint() + "XXXXXX");
82 char *atemp = ::strdup( apath.asString().c_str());
83 char *atest = NULL;
84 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
85 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
86 {
87 WAR << "attach point " << ainfo.path()
88 << " is not useable for " << url_r.url().getScheme() << endl;
89 setAttachPoint("", true);
90 }
91 else if( atest != NULL)
92 ::rmdir(atest);
93
94 if( atemp != NULL)
95 ::free(atemp);
96 }
97 }
98
100
102 {
103 if ( !zyppng::NetworkRequestDispatcher::supportsProtocol ( url ) )
104 {
105 std::string msg("Unsupported protocol '");
106 msg += url.getScheme();
107 msg += "'";
109 }
110 }
111
113 {
114 // clear effective settings
115 _mirrorSettings.clear();
116 }
117
119
120 void MediaCurl2::releaseFrom( const std::string & ejectDev )
121 {
122 disconnect();
123 }
124
126
127 void MediaCurl2::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
128 {
129
130 const auto &filename = srcFile.filename();
131
132 // Optional files will send no report until data are actually received (we know it exists).
133 OptionalDownloadProgressReport reportfilter( srcFile.optional() );
135
136 for ( unsigned mirr = 0; mirr < _urls.size(); ++mirr ) {
137
138 MIL << "Trying to fetch file " << srcFile << " with mirror " << _urls[mirr].url() << std::endl;
139 Url fileurl(getFileUrl(mirr, filename));
140
141 try
142 {
143 if(!_urls[mirr].url().isValid())
145
146 if(_urls[mirr].url().getHost().empty())
148
149
150 Pathname dest = target.absolutename();
151 if( assert_dir( dest.dirname() ) ) {
152 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
153 ZYPP_THROW( MediaSystemException(fileurl, "System error on " + dest.dirname().asString()) );
154 }
155
156 ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) }; {
157 AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
158 if( ! buf ) {
159 ERR << "out of memory for temp file name" << endl;
160 ZYPP_THROW(MediaSystemException(fileurl, "out of memory for temp file name"));
161 }
162
163 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
164 if( tmp_fd == -1 ) {
165 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
167 }
168 destNew = ManagedFile( (*buf), filesystem::unlink );
169 }
170
171 DBG << "dest: " << dest << endl;
172 DBG << "temp: " << destNew << endl;
173 #if 0
174 Not implemented here yet because NetworkRequest can not do IFMODSINCE yet
175 // set IFMODSINCE time condition (no download if not modified)
176 if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
177 {
178 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
179 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
180 }
181 else
182 {
183 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
184 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
185 }
186 #endif
187
188 DBG << srcFile.filename().asString() << endl;
189 DBG << "URL: " << fileurl.asString() << endl;
190
191 // Use URL without options and without username and passwd
192 // (some proxies dislike them in the URL).
193 // Curl seems to need the just scheme, hostname and a path;
194 // the rest was already passed as curl options (in attachTo).
195 Url curlUrl( clearQueryString(fileurl) );
196
197 RequestData r;
198 r._mirrorIdx = mirr;
199 r._req = std::make_shared<zyppng::NetworkRequest>( curlUrl, destNew, zyppng::NetworkRequest::WriteShared /*do not truncate*/ );
200 r._req->transferSettings() = _mirrorSettings[mirr];
201 r._req->setExpectedFileSize ( srcFile.downloadSize () );
202
203 bool done = false;
204 #ifdef ENABLE_ZCHUNK_COMPRESSION
205 done = const_cast<MediaCurl2*>(this)->tryZchunk(r, srcFile, destNew, report);
206 #endif
207 if ( !done ) {
208 r._req->resetRequestRanges();
209 const_cast<MediaCurl2 *>(this)->executeRequest ( r, &report );
210 }
211
212 #if 0
213 Also disabled IFMODSINCE code, see above while not yet implemented here
214 #if CURLVERSION_AT_LEAST(7,19,4)
215 // bnc#692260: If the client sends a request with an If-Modified-Since header
216 // with a future date for the server, the server may respond 200 sending a
217 // zero size file.
218 // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
219 if ( ftell(file) == 0 && ret == 0 )
220 {
221 long httpReturnCode = 33;
222 if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
223 {
224 long conditionUnmet = 33;
225 if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
226 {
227 WAR << "TIMECONDITION unmet - retry without." << endl;
228 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
229 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
230 ret = executeCurl();
231 }
232 }
233 }
234 #endif
235 #endif
236
237
238 // apply umask
239 if ( ::chmod( destNew->c_str(), filesystem::applyUmaskTo( 0644 ) ) )
240 {
241 ERR << "Failed to chmod file " << destNew << endl;
242 }
243
244 // move the temp file into dest
245 if ( rename( destNew, dest ) != 0 ) {
246 ERR << "Rename failed" << endl;
248 }
249 destNew.resetDispose(); // no more need to unlink it
250
251 DBG << "done: " << PathInfo(dest) << endl;
252
253 break; // success!
254 }
255 catch (MediaException & excpt_r)
256 {
257 // check if we can retry on the next mirror
258 if( !canTryNextMirror ( excpt_r ) || ( mirr+1 >= _urls.size() ) ) {
259 // rewrite the exception to contain the correct pathname and url
260 // the executeRequest implementation just emits the full Url in the exception
261 if ( typeid(excpt_r) == typeid( MediaFileNotFoundException ) ) {
262 ZYPP_CAUGHT(excpt_r);
263 ZYPP_THROW( MediaFileNotFoundException( url(), filename ) );
264 }
265 ZYPP_RETHROW(excpt_r);
266 }
267 ZYPP_CAUGHT(excpt_r);
268 continue;
269 }
270 }
271 }
272
273 bool MediaCurl2::getDoesFileExist( const Pathname & filename ) const
274 {
275 DBG << filename.asString() << endl;
276
277 std::exception_ptr lastErr;
278 for ( unsigned mirr = 0; mirr < _urls.size(); ++mirr ) {
279 if( !_urls[mirr].url().isValid() )
281
282 if( _urls[mirr].url().getHost().empty() )
284
285 Url url(getFileUrl(mirr, filename));
286
287 DBG << "URL: " << url.asString() << endl;
288
289 // Use URL without options and without username and passwd
290 // (some proxies dislike them in the URL).
291 // Curl seems to need the just scheme, hostname and a path;
292 // the rest was already passed as curl options (in attachTo).
293 Url curlUrl( clearQueryString(url) );
294
295 RequestData r;
296 r._mirrorIdx = mirr;
297 r._req = std::make_shared<zyppng::NetworkRequest>( curlUrl, "/dev/null" );
298 r._req->setOptions ( zyppng::NetworkRequest::HeadRequest ); // just check for existance
299 r._req->transferSettings() = _mirrorSettings[mirr];
300
301 // as we are not having user interaction, the user can't cancel
302 // the file existence checking, a callback or timeout return code
303 // will be always a timeout.
304 try {
305 const_cast<MediaCurl2*>(this)->executeRequest ( r );
306
307 } catch ( const MediaException &e ) {
308 // check if we can retry on the next mirror
309 if( !canTryNextMirror ( e ) ) {
310 ZYPP_RETHROW(e);
311 }
312 lastErr = ZYPP_FWD_CURRENT_EXCPT();
313 continue;
314 }
315 // exists
316 return ( !r._req->hasError() );
317 }
318
319 if ( lastErr ) {
320 try {
321 ZYPP_RETHROW (lastErr);
322 } catch ( const MediaFileNotFoundException &e ) {
323 // if the file did not exist then we can return false
324 return false;
325 }
326 }
327 // we probably never had mirrors, otherwise we either had an error or a success.
328 return false;
329 }
330
332 {
333#ifdef ENABLE_ZCHUNK_COMPRESSION
334
335 // HERE add zchunk logic if required
336 if ( !srcFile.deltafile().empty()
338 && srcFile.headerSize () > 0 ) {
339
340 // first fetch the zck header
341 std::optional<zypp::Digest> digest;
342 UByteArray sum;
343
344 const auto &headerSum = srcFile.headerChecksum();
345 if ( !headerSum.empty () ) {
346 digest = zypp::Digest();
347 if ( !digest->create( headerSum.type() ) ) {
348 ERR << "Unknown header checksum type " << headerSum.type() << std::endl;
349 return false;
350 }
351 sum = zypp::Digest::hexStringToUByteArray( headerSum.checksum() );
352 }
353
354 reqData._req->addRequestRange( 0, srcFile.headerSize(), std::move(digest), sum );
355 executeRequest ( reqData, nullptr );
356
357 reqData._req->resetRequestRanges();
358
359 auto res = zyppng::ZckHelper::prepareZck( srcFile.deltafile(), target, srcFile.downloadSize() );
360 switch(res._code) {
362 ERR << "Failed to setup zchunk because of: " << res._message << std::endl;
363 return false;
364 }
366 return true; // already done
368 ZYPP_THROW( MediaFileSizeExceededException( reqData._req->url(), srcFile.downloadSize(), res._message ));
370 break;
371 }
372
373 for ( const auto &block : res._blocks ) {
374 if ( block._checksum.size() && block._chksumtype.size() ) {
375 std::optional<zypp::Digest> dig = zypp::Digest();
376 if ( !dig->create( block._chksumtype ) ) {
377 WAR_MEDIA << "Trying to create Digest with chksum type " << block._chksumtype << " failed " << std::endl;
378 return false;
379 }
380
382 DBG_MEDIA << "Starting block " << block._start << " with checksum " << zypp::Digest::digestVectorToString( block._checksum ) << "." << std::endl;
383 reqData._req->addRequestRange( block._start, block._len, std::move(dig), block._checksum, {}, block._relevantDigestLen, block._chksumPad );
384 }
385 };
386
387 executeRequest ( reqData, &report );
388
389 //we might have the file ready
390 std::string err;
392 ERR << "ZCK failed with error: " << err << std::endl;
393 return false;
394 }
395 return true;
396 }
397#endif
398 return false;
399 }
400
402 {
403 const auto &authCb = [&]( const zypp::Url &, TransferSettings &settings, const std::string & availAuthTypes, bool firstTry, bool &canContinue ) {
404 if ( authenticate( _urls[reqData._mirrorIdx].url(), _mirrorSettings[reqData._mirrorIdx], availAuthTypes, firstTry ) ) {
405 settings = _mirrorSettings[reqData._mirrorIdx];
406 canContinue = true;
407 return;
408 }
409 canContinue = false;
410 };
411
412 auto conn = _executor->sigAuthRequired().connect (authCb);
413 zypp_defer {
414 conn.disconnect ();
415 };
416
417 _executor->executeRequest ( reqData._req, report );
418 }
419 } // namespace media
420} // namespace zypp
421//
Compute Message Digests (MD5, SHA1 etc)
Definition Digest.h:38
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition Digest.cc:244
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const ByteCount & downloadSize() const
The size of the resource on the server.
const Pathname & filename() const
The path to the resource on the medium.
const Pathname & deltafile() const
The existing deltafile that can be used to reduce download size ( zchunk or metalink )
const ByteCount & headerSize() const
The size of the header prepending the resource (e.g.
const CheckSum & headerChecksum() const
The checksum of the header prepending the resource (e.g.
Url manipulation class.
Definition Url.h:93
std::string asString() const
Returns a default string representation of the Url object.
Definition Url.cc:515
Wrapper class for stat/lstat.
Definition PathInfo.h:226
const Pathname & path() const
Return current Pathname.
Definition PathInfo.h:251
Pathname dirname() const
Return all but the last component od this path.
Definition Pathname.h:126
const std::string & asString() const
String representation.
Definition Pathname.h:93
bool empty() const
Test for an empty path.
Definition Pathname.h:116
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
void disconnectFrom() override
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
MediaCurl2(const MediaUrl &url_r, const std::vector< MediaUrl > &mirrors_r, const Pathname &attach_point_hint_r)
Definition MediaCurl2.cc:68
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
void executeRequest(RequestData &reqData, callback::SendReport< DownloadProgressReport > *report=nullptr)
internal::MediaNetworkRequestExecutorRef _executor
Definition MediaCurl2.h:103
bool tryZchunk(RequestData &reqData, const OnMediaLocation &srcFile, const Pathname &target, callback::SendReport< DownloadProgressReport > &report)
void checkProtocol(const Url &url) const override
check the url is supported by the curl library
Just inherits Exception to separate media exceptions.
Url url() const
Primary Url used.
const MediaUrl _url
Primary Url.
void disconnect()
Use concrete handler to isconnect media.
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname attachPoint() const
Return the currently used attach point.
const std::vector< MediaUrl > _urls
All mirrors including the primary.
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
static bool canTryNextMirror(const Excpt &excpt_r)
MediaNetworkCommonHandler(const MediaUrl &url_r, const std::vector< MediaUrl > &mirrors_r, const Pathname &attach_point_r, const Pathname &urlpath_below_attachpoint_r, const bool does_download_r)
Url getFileUrl(int mirrorIdx, const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
std::vector< TransferSettings > _mirrorSettings
Holds transfer setting.
static bool validateZckFile(const zypp::Pathname &file, std::string &error)
Definition zckhelper.cc:169
static bool isZchunkFile(const zypp::Pathname &file)
Definition zckhelper.cc:21
static PrepareResult prepareZck(const zypp::Pathname &delta, const zypp::Pathname &target, const zypp::ByteCount &expectedFileSize)
Definition zckhelper.cc:34
#define WAR_MEDIA
#define DBG_MEDIA
Definition Arch.h:364
const long & ZYPP_MEDIA_CURL_DEBUG()
const long& for setting CURLOPT_DEBUGDATA Returns a reference to a static variable,...
Definition curlhelper.cc:36
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition PathInfo.h:805
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:705
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition ManagedFile.h:27
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
AutoDispose<int> calling ::close
zyppng::NetworkRequestRef _req
Definition MediaCurl2.h:96
#define zypp_defer
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition Exception.h:479
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:475
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition Exception.h:471
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:459
#define DBG
Definition Logger.h:99
#define MIL
Definition Logger.h:100
#define ERR
Definition Logger.h:102
#define WAR
Definition Logger.h:101
Interface to gettext.