]>
git.saurik.com Git - apt.git/blob - ftparchive/writer.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: writer.cc,v 1.7 2003/02/10 07:34:41 doogie Exp $
4 /* ######################################################################
8 The file writer classes. These write various types of output, sources,
11 ##################################################################### */
13 // Include Files /*{{{*/
15 #pragma implementation "writer.h"
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/configuration.h>
24 #include <apt-pkg/md5.h>
25 #include <apt-pkg/deblistparser.h>
27 #include <sys/types.h>
33 #include "apt-ftparchive.h"
34 #include "multicompress.h"
38 FTWScanner
*FTWScanner::Owner
;
40 // SetTFRewriteData - Helper for setting rewrite lists /*{{{*/
41 // ---------------------------------------------------------------------
43 inline void SetTFRewriteData(struct TFRewriteData
&tfrd
,
46 const char *newtag
= 0)
49 tfrd
.Rewrite
= rewrite
;
54 // FTWScanner::FTWScanner - Constructor /*{{{*/
55 // ---------------------------------------------------------------------
57 FTWScanner::FTWScanner()
60 NoLinkAct
= !_config
->FindB("APT::FTPArchive::DeLinkAct",true);
64 long PMax
= pathconf(".",_PC_PATH_MAX
);
66 RealPath
= new char[PMax
];
69 // FTWScanner::Scanner - FTW Scanner /*{{{*/
70 // ---------------------------------------------------------------------
71 /* This is the FTW scanner, it processes each directory element in the
73 int FTWScanner::Scanner(const char *File
,const struct stat
*sb
,int Flag
)
78 ioprintf(c1out
, _("W: Unable to read directory %s\n"), File
);
83 ioprintf(c1out
, _("W: Unable to stat %s\n"), File
);
88 // See if it is a .deb
93 for (; Owner
->Ext
[CurExt
] != 0; CurExt
++)
94 if (strcmp(File
+strlen(File
)-strlen(Owner
->Ext
[CurExt
]),
95 Owner
->Ext
[CurExt
]) == 0)
97 if (Owner
->Ext
[CurExt
] == 0)
100 /* Process it. If the file is a link then resolve it into an absolute
101 name.. This works best if the directory components the scanner are
102 given are not links themselves. */
104 Owner
->OriginalPath
= File
;
105 if (Owner
->RealPath
!= 0 && readlink(File
,Jnk
,sizeof(Jnk
)) != -1 &&
106 realpath(File
,Owner
->RealPath
) != 0)
107 Owner
->DoPackage(Owner
->RealPath
);
109 Owner
->DoPackage(File
);
111 if (_error
->empty() == false)
113 // Print any errors or warnings found
115 bool SeenPath
= false;
116 while (_error
->empty() == false)
120 bool Type
= _error
->PopMessage(Err
);
122 cerr
<< _("E: ") << Err
<< endl
;
124 cerr
<< _("W: ") << Err
<< endl
;
126 if (Err
.find(File
) != string::npos
)
130 if (SeenPath
== false)
131 cerr
<< _("E: Errors apply to file ") << "'" << File
<< "'" << endl
;
138 // FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
139 // ---------------------------------------------------------------------
141 bool FTWScanner::RecursiveScan(string Dir
)
143 /* If noprefix is set then jam the scan root in, so we don't generate
144 link followed paths out of control */
145 if (InternalPrefix
.empty() == true)
147 if (realpath(Dir
.c_str(),RealPath
) == 0)
148 return _error
->Errno("realpath",_("Failed to resolve %s"),Dir
.c_str());
149 InternalPrefix
= RealPath
;
152 // Do recursive directory searching
154 int Res
= ftw(Dir
.c_str(),Scanner
,30);
156 // Error treewalking?
159 if (_error
->PendingError() == false)
160 _error
->Errno("ftw",_("Tree walking failed"));
167 // FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
168 // ---------------------------------------------------------------------
169 /* This is an alternative to using FTW to locate files, it reads the list
170 of files from another file. */
171 bool FTWScanner::LoadFileList(string Dir
,string File
)
173 /* If noprefix is set then jam the scan root in, so we don't generate
174 link followed paths out of control */
175 if (InternalPrefix
.empty() == true)
177 if (realpath(Dir
.c_str(),RealPath
) == 0)
178 return _error
->Errno("realpath",_("Failed to resolve %s"),Dir
.c_str());
179 InternalPrefix
= RealPath
;
183 FILE *List
= fopen(File
.c_str(),"r");
185 return _error
->Errno("fopen",_("Failed to open %s"),File
.c_str());
187 /* We are a tad tricky here.. We prefix the buffer with the directory
188 name, that way if we need a full path with just use line.. Sneaky and
192 if (Dir
.empty() == true || Dir
.end()[-1] != '/')
193 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s/",Dir
.c_str());
195 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s",Dir
.c_str());
196 while (fgets(FileStart
,sizeof(Line
) - (FileStart
- Line
),List
) != 0)
198 char *FileName
= _strstrip(FileStart
);
199 if (FileName
[0] == 0)
202 if (FileName
[0] != '/')
204 if (FileName
!= FileStart
)
205 memmove(FileStart
,FileName
,strlen(FileStart
));
211 if (stat(FileName
,&St
) != 0)
214 if (Scanner(FileName
,&St
,Flag
) != 0)
222 // FTWScanner::Delink - Delink symlinks /*{{{*/
223 // ---------------------------------------------------------------------
225 bool FTWScanner::Delink(string
&FileName
,const char *OriginalPath
,
226 unsigned long &DeLinkBytes
,
229 // See if this isn't an internaly prefix'd file name.
230 if (InternalPrefix
.empty() == false &&
231 InternalPrefix
.length() < FileName
.length() &&
232 stringcmp(FileName
.begin(),FileName
.begin() + InternalPrefix
.length(),
233 InternalPrefix
.begin(),InternalPrefix
.end()) != 0)
235 if (DeLinkLimit
!= 0 && DeLinkBytes
/1024 < DeLinkLimit
)
237 // Tidy up the display
238 if (DeLinkBytes
== 0)
242 ioprintf(c1out
, _(" DeLink %s [%s]\n"), (OriginalPath
+ InternalPrefix
.length()),
243 SizeToStr(St
.st_size
).c_str());
246 if (NoLinkAct
== false)
249 if (readlink(OriginalPath
,OldLink
,sizeof(OldLink
)) == -1)
250 _error
->Errno("readlink",_("Failed to readlink %s"),OriginalPath
);
253 if (unlink(OriginalPath
) != 0)
254 _error
->Errno("unlink",_("Failed to unlink %s"),OriginalPath
);
257 if (link(FileName
.c_str(),OriginalPath
) != 0)
259 // Panic! Restore the symlink
260 symlink(OldLink
,OriginalPath
);
261 return _error
->Errno("link",_("*** Failed to link %s to %s"),
269 DeLinkBytes
+= St
.st_size
;
270 if (DeLinkBytes
/1024 >= DeLinkLimit
)
271 ioprintf(c1out
, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes
).c_str());
274 FileName
= OriginalPath
;
280 // FTWScanner::SetExts - Set extensions to support /*{{{*/
281 // ---------------------------------------------------------------------
283 bool FTWScanner::SetExts(string Vals
)
286 TmpExt
= new char[Vals
.length()+1];
287 strcpy(TmpExt
,Vals
.c_str());
288 return TokSplitString(' ',TmpExt
,(char **)Ext
,sizeof(Ext
)/sizeof(Ext
[0]));
292 // PackagesWriter::PackagesWriter - Constructor /*{{{*/
293 // ---------------------------------------------------------------------
295 PackagesWriter::PackagesWriter(string DB
,string Overrides
,string ExtOverrides
) :
296 Db(DB
),Stats(Db
.Stats
)
303 // Process the command line options
304 DoMD5
= _config
->FindB("APT::FTPArchive::MD5",true);
305 DoContents
= _config
->FindB("APT::FTPArchive::Contents",true);
306 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
308 if (Db
.Loaded() == false)
311 // Read the override file
312 if (Overrides
.empty() == false && Over
.ReadOverride(Overrides
) == false)
317 if (ExtOverrides
.empty() == false)
318 Over
.ReadExtraOverride(ExtOverrides
);
320 _error
->DumpErrors();
323 // PackagesWriter::DoPackage - Process a single package /*{{{*/
324 // ---------------------------------------------------------------------
325 /* This method takes a package and gets its control information and
326 MD5 then writes out a control record with the proper fields rewritten
327 and the path/size/hash appended. */
328 bool PackagesWriter::DoPackage(string FileName
)
331 FileFd
F(FileName
,FileFd::ReadOnly
);
332 if (_error
->PendingError() == true)
335 // Stat the file for later
337 if (fstat(F
.Fd(),&St
) != 0)
338 return _error
->Errno("fstat",_("Failed to stat %s"),FileName
.c_str());
340 // Pull all the data we need form the DB
342 if (Db
.SetFile(FileName
,St
,&F
) == false ||
343 Db
.LoadControl() == false ||
344 (DoContents
== true && Db
.LoadContents(true) == false) ||
345 (DoMD5
== true && Db
.GetMD5(MD5Res
,false) == false))
348 if (Delink(FileName
,OriginalPath
,Stats
.DeLinkBytes
,St
) == false)
351 // Lookup the overide information
352 pkgTagSection
&Tags
= Db
.Control
.Section
;
353 string Package
= Tags
.FindS("Package");
355 Override::Item
*OverItem
= Over
.GetItem(Package
);
357 if (Package
.empty() == true)
358 return _error
->Error(_("Archive had no package field"));
360 // If we need to do any rewriting of the header do it now..
363 if (NoOverride
== false)
366 ioprintf(c1out
, _(" %s has no override entry\n"), Package
.c_str());
370 Tmp
.FieldOverride
["Section"] = Tags
.FindS("Section");
371 Tmp
.Priority
= Tags
.FindS("Priority");
375 sprintf(Size
,"%lu",St
.st_size
);
377 // Strip the DirStrip prefix from the FileName and add the PathPrefix
379 if (DirStrip
.empty() == false &&
380 FileName
.length() > DirStrip
.length() &&
381 stringcmp(FileName
.begin(),FileName
.begin() + DirStrip
.length(),
382 DirStrip
.begin(),DirStrip
.end()) == 0)
383 NewFileName
= string(FileName
.begin() + DirStrip
.length(),FileName
.end());
385 NewFileName
= FileName
;
386 if (PathPrefix
.empty() == false)
387 NewFileName
= flCombine(PathPrefix
,NewFileName
);
389 // This lists all the changes to the fields we are going to make.
390 // (7 hardcoded + maintainer + suggests + end marker)
391 TFRewriteData Changes
[6+2+OverItem
->FieldOverride
.size()+1];
393 unsigned int End
= 0;
394 SetTFRewriteData(Changes
[End
++], "Size", Size
);
395 SetTFRewriteData(Changes
[End
++], "MD5sum", MD5Res
.c_str());
396 SetTFRewriteData(Changes
[End
++], "Filename", NewFileName
.c_str());
397 SetTFRewriteData(Changes
[End
++], "Priority", OverItem
->Priority
.c_str());
398 SetTFRewriteData(Changes
[End
++], "Status", 0);
399 SetTFRewriteData(Changes
[End
++], "Optional", 0);
401 // Rewrite the maintainer field if necessary
403 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
404 if (MaintFailed
== true)
406 if (NoOverride
== false)
409 ioprintf(c1out
, _(" %s maintainer is %s not %s\n"),
410 Package
.c_str(), Tags
.FindS("Maintainer").c_str(), OverItem
->OldMaint
.c_str());
414 if (NewMaint
.empty() == false)
415 SetTFRewriteData(Changes
[End
++], "Maintainer", NewMaint
.c_str());
417 /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
418 dpkg-scanpackages does.. Well sort of. dpkg-scanpackages just does renaming
419 but dpkg does this append bit. So we do the append bit, at least that way the
420 status file and package file will remain similar. There are other transforms
421 but optional is the only legacy one still in use for some lazy reason. */
422 string OptionalStr
= Tags
.FindS("Optional");
423 if (OptionalStr
.empty() == false)
425 if (Tags
.FindS("Suggests").empty() == false)
426 OptionalStr
= Tags
.FindS("Suggests") + ", " + OptionalStr
;
427 SetTFRewriteData(Changes
[End
++], "Suggests", OptionalStr
.c_str());
430 for (map
<string
,string
>::iterator I
= OverItem
->FieldOverride
.begin();
431 I
!= OverItem
->FieldOverride
.end(); I
++)
432 SetTFRewriteData(Changes
[End
++],I
->first
.c_str(),I
->second
.c_str());
434 SetTFRewriteData(Changes
[End
++], 0, 0);
436 // Rewrite and store the fields.
437 if (TFRewrite(Output
,Tags
,TFRewritePackageOrder
,Changes
) == false)
439 fprintf(Output
,"\n");
445 // SourcesWriter::SourcesWriter - Constructor /*{{{*/
446 // ---------------------------------------------------------------------
448 SourcesWriter::SourcesWriter(string BOverrides
,string SOverrides
,
458 // Process the command line options
459 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
461 // Read the override file
462 if (BOverrides
.empty() == false && BOver
.ReadOverride(BOverrides
) == false)
467 if (ExtOverrides
.empty() == false)
468 SOver
.ReadExtraOverride(ExtOverrides
);
470 if (SOverrides
.empty() == false && FileExists(SOverrides
) == true)
471 SOver
.ReadOverride(SOverrides
,true);
474 // SourcesWriter::DoPackage - Process a single package /*{{{*/
475 // ---------------------------------------------------------------------
477 bool SourcesWriter::DoPackage(string FileName
)
480 FileFd
F(FileName
,FileFd::ReadOnly
);
481 if (_error
->PendingError() == true)
484 // Stat the file for later
486 if (fstat(F
.Fd(),&St
) != 0)
487 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
489 if (St
.st_size
> 128*1024)
490 return _error
->Error("DSC file '%s' is too large!",FileName
.c_str());
492 if (BufSize
< (unsigned)St
.st_size
+1)
494 BufSize
= St
.st_size
+1;
495 Buffer
= (char *)realloc(Buffer
,St
.st_size
+1);
498 if (F
.Read(Buffer
,St
.st_size
) == false)
502 char *Start
= Buffer
;
503 char *BlkEnd
= Buffer
+ St
.st_size
;
505 MD5
.Add((unsigned char *)Start
,BlkEnd
- Start
);
507 // Add an extra \n to the end, just in case
510 /* Remove the PGP trailer. Some .dsc's have this without a blank line
512 const char *Key
= "-----BEGIN PGP SIGNATURE-----";
513 for (char *MsgEnd
= Start
; MsgEnd
< BlkEnd
- strlen(Key
) -1; MsgEnd
++)
515 if (*MsgEnd
== '\n' && strncmp(MsgEnd
+1,Key
,strlen(Key
)) == 0)
522 /* Read records until we locate the Source record. This neatly skips the
523 GPG header (which is RFC822 formed) without any trouble. */
528 if (Tags
.Scan(Start
,BlkEnd
- Start
) == false)
529 return _error
->Error("Could not find a record in the DSC '%s'",FileName
.c_str());
530 if (Tags
.Find("Source",Pos
) == true)
532 Start
+= Tags
.size();
537 // Lookup the overide information, finding first the best priority.
540 string Bins
= Tags
.FindS("Binary");
541 Override::Item
*OverItem
= 0;
542 if (Bins
.empty() == false && Bins
.length() < sizeof(Buffer
))
544 strcpy(Buffer
,Bins
.c_str());
546 // Ignore too-long errors.
548 TokSplitString(',',Buffer
,BinList
,sizeof(BinList
)/sizeof(BinList
[0]));
550 // Look at all the binaries
551 unsigned char BestPrioV
= pkgCache::State::Extra
;
552 for (unsigned I
= 0; BinList
[I
] != 0; I
++)
554 Override::Item
*Itm
= BOver
.GetItem(BinList
[I
]);
560 unsigned char NewPrioV
= debListParser::GetPrio(Itm
->Priority
);
561 if (NewPrioV
< BestPrioV
|| BestPrio
.empty() == true)
563 BestPrioV
= NewPrioV
;
564 BestPrio
= Itm
->Priority
;
569 // If we need to do any rewriting of the header do it now..
573 if (NoOverride
== false)
576 ioprintf(c1out
, _(" %s has no override entry\n"), Tags
.FindS("Source").c_str());
582 Override::Item
*SOverItem
= SOver
.GetItem(Tags
.FindS("Source"));
585 SOverItem
= BOver
.GetItem(Tags
.FindS("Source"));
587 SOverItem
= OverItem
;
590 // Add the dsc to the files hash list
592 snprintf(Files
,sizeof(Files
),"\n %s %lu %s\n %s",
593 string(MD5
.Result()).c_str(),St
.st_size
,
594 flNotDir(FileName
).c_str(),
595 Tags
.FindS("Files").c_str());
597 // Strip the DirStrip prefix from the FileName and add the PathPrefix
599 if (DirStrip
.empty() == false &&
600 FileName
.length() > DirStrip
.length() &&
601 stringcmp(DirStrip
,OriginalPath
,OriginalPath
+ DirStrip
.length()) == 0)
602 NewFileName
= string(OriginalPath
+ DirStrip
.length());
604 NewFileName
= OriginalPath
;
605 if (PathPrefix
.empty() == false)
606 NewFileName
= flCombine(PathPrefix
,NewFileName
);
608 string Directory
= flNotFile(OriginalPath
);
609 string Package
= Tags
.FindS("Source");
611 // Perform the delinking operation over all of the files
613 const char *C
= Files
;
614 for (;isspace(*C
); C
++);
617 // Parse each of the elements
618 if (ParseQuoteWord(C
,ParseJnk
) == false ||
619 ParseQuoteWord(C
,ParseJnk
) == false ||
620 ParseQuoteWord(C
,ParseJnk
) == false)
621 return _error
->Error("Error parsing file record");
624 string OriginalPath
= Directory
+ ParseJnk
;
625 if (RealPath
!= 0 && readlink(OriginalPath
.c_str(),Jnk
,sizeof(Jnk
)) != -1 &&
626 realpath(OriginalPath
.c_str(),RealPath
) != 0)
628 string RP
= RealPath
;
629 if (Delink(RP
,OriginalPath
.c_str(),Stats
.DeLinkBytes
,St
) == false)
634 Directory
= flNotFile(NewFileName
);
635 if (Directory
.length() > 2)
636 Directory
.erase(Directory
.end()-1);
638 // This lists all the changes to the fields we are going to make.
639 // (5 hardcoded + maintainer + end marker)
640 TFRewriteData Changes
[5+1+SOverItem
->FieldOverride
.size()+1];
642 unsigned int End
= 0;
643 SetTFRewriteData(Changes
[End
++],"Source",Package
.c_str(),"Package");
644 SetTFRewriteData(Changes
[End
++],"Files",Files
);
645 if (Directory
!= "./")
646 SetTFRewriteData(Changes
[End
++],"Directory",Directory
.c_str());
647 SetTFRewriteData(Changes
[End
++],"Priority",BestPrio
.c_str());
648 SetTFRewriteData(Changes
[End
++],"Status",0);
650 // Rewrite the maintainer field if necessary
652 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
653 if (MaintFailed
== true)
655 if (NoOverride
== false)
658 ioprintf(c1out
, _(" %s maintainer is %s not %s\n"), Package
.c_str(),
659 Tags
.FindS("Maintainer").c_str(), OverItem
->OldMaint
.c_str());
662 if (NewMaint
.empty() == false)
663 SetTFRewriteData(Changes
[End
++], "Maintainer", NewMaint
.c_str());
665 for (map
<string
,string
>::iterator I
= SOverItem
->FieldOverride
.begin();
666 I
!= SOverItem
->FieldOverride
.end(); I
++)
667 SetTFRewriteData(Changes
[End
++],I
->first
.c_str(),I
->second
.c_str());
669 SetTFRewriteData(Changes
[End
++], 0, 0);
671 // Rewrite and store the fields.
672 if (TFRewrite(Output
,Tags
,TFRewriteSourceOrder
,Changes
) == false)
674 fprintf(Output
,"\n");
682 // ContentsWriter::ContentsWriter - Constructor /*{{{*/
683 // ---------------------------------------------------------------------
685 ContentsWriter::ContentsWriter(string DB
) :
686 Db(DB
), Stats(Db
.Stats
)
694 // ContentsWriter::DoPackage - Process a single package /*{{{*/
695 // ---------------------------------------------------------------------
696 /* If Package is the empty string the control record will be parsed to
697 determine what the package name is. */
698 bool ContentsWriter::DoPackage(string FileName
,string Package
)
701 FileFd
F(FileName
,FileFd::ReadOnly
);
702 if (_error
->PendingError() == true)
705 // Stat the file for later
707 if (fstat(F
.Fd(),&St
) != 0)
708 return _error
->Errno("fstat","Failed too stat %s",FileName
.c_str());
711 if (Db
.SetFile(FileName
,St
,&F
) == false ||
712 Db
.LoadContents(false) == false)
715 // Parse the package name
716 if (Package
.empty() == true)
718 if (Db
.LoadControl() == false)
720 Package
= Db
.Control
.Section
.FindS("Package");
723 Db
.Contents
.Add(Gen
,Package
);
728 // ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
729 // ---------------------------------------------------------------------
731 bool ContentsWriter::ReadFromPkgs(string PkgFile
,string PkgCompress
)
733 MultiCompress
Pkgs(PkgFile
,PkgCompress
,0,false);
734 if (_error
->PendingError() == true)
737 // Open the package file
740 if (Pkgs
.OpenOld(CompFd
,Proc
) == false)
744 FileFd
Fd(CompFd
,false);
745 pkgTagFile
Tags(&Fd
);
746 if (_error
->PendingError() == true)
748 Pkgs
.CloseOld(CompFd
,Proc
);
753 pkgTagSection Section
;
754 while (Tags
.Step(Section
) == true)
756 string File
= flCombine(Prefix
,Section
.FindS("FileName"));
757 string Package
= Section
.FindS("Section");
758 if (Package
.empty() == false && Package
.end()[-1] != '/')
761 Package
+= Section
.FindS("Package");
764 Package
+= Section
.FindS("Package");
766 DoPackage(File
,Package
);
767 if (_error
->empty() == false)
769 _error
->Error("Errors apply to file '%s'",File
.c_str());
770 _error
->DumpErrors();
774 // Tidy the compressor
775 if (Pkgs
.CloseOld(CompFd
,Proc
) == false)